
'''simple docstring'''





import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp





def 							_lowerCamelCase			(					lowerCamelCase_:  Tuple				,	lowerCamelCase_:  Union[str, Any]				,	lowerCamelCase_:  Tuple				,	lowerCamelCase_:  Any				,	lowerCamelCase_:  int = 4096		):



				'''simple docstring'''







				A     :  Optional[Any]   =				key.shape[-3:]
				A     :  List[str]   =				value.shape[-1]
				A     :  Optional[Any]   =				min(lowerCamelCase_				,	lowerCamelCase_		)
				A     :  List[str]   =				query / jnp.sqrt(lowerCamelCase_		)

				@functools.partial(jax.checkpoint				,	prevent_cse=lowerCamelCase_		)
				def summarize_chunk(lowerCamelCase_:  int				,	lowerCamelCase_:  str				,	lowerCamelCase_:  Dict		):
								A     :  Union[str, Any]   =				jnp.einsum('''...qhd,...khd->...qhk'''				,	lowerCamelCase_				,	lowerCamelCase_				,	precision=lowerCamelCase_		)

								A     :  List[str]   =				jnp.max(lowerCamelCase_				,	axis=-1				,	keepdims=lowerCamelCase_		)
								A     :  List[Any]   =				jax.lax.stop_gradient(lowerCamelCase_		)
								A     :  str   =				jnp.exp(attn_weights - max_score		)

								A     :  Dict   =				jnp.einsum('''...vhf,...qhv->...qhf'''				,	lowerCamelCase_				,	lowerCamelCase_				,	precision=lowerCamelCase_		)
								A     :  str   =				jnp.einsum('''...qhk->...qh'''				,	lowerCamelCase_		)

								return (exp_values, exp_weights.sum(axis=-1		), max_score)

				def chunk_scanner(lowerCamelCase_:  Any		):
								# julienne key array
								A     :  Optional[int]   =				jax.lax.dynamic_slice(
								    operand=lowerCamelCase_				,	start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]				,	slice_sizes=list(key.shape[:-3]		) + [key_chunk_size, num_heads, k_features]				,	)

								# julienne value array
								A     :  List[str]   =				jax.lax.dynamic_slice(
								    operand=lowerCamelCase_				,	start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]				,	slice_sizes=list(value.shape[:-3]		) + [key_chunk_size, num_heads, v_features]				,	)

								return summarize_chunk(lowerCamelCase_				,	lowerCamelCase_				,	lowerCamelCase_		)

				A     :  int   =				jax.lax.map(f=lowerCamelCase_				,	xs=jnp.arange(0				,	lowerCamelCase_				,	lowerCamelCase_		)		)

				A     :  Tuple   =				jnp.max(lowerCamelCase_				,	axis=0				,	keepdims=lowerCamelCase_		)
				A     :  int   =				jnp.exp(chunk_max - global_max		)

				chunk_values *= jnp.expand_dims(lowerCamelCase_				,	axis=-1		)
				chunk_weights *= max_diffs

				A     :  List[str]   =				chunk_values.sum(axis=0		)
				A     :  Tuple   =				jnp.expand_dims(lowerCamelCase_				,	-1		).sum(axis=0		)

				return all_values / all_weights





def 							_lowerCamelCase			(					lowerCamelCase_:  Optional[Any]				,	lowerCamelCase_:  Optional[Any]				,	lowerCamelCase_:  Optional[int]				,	lowerCamelCase_:  Union[str, Any]=jax.lax.Precision.HIGHEST				,	lowerCamelCase_:  int = 1024				,	lowerCamelCase_:  int = 4096		):



				'''simple docstring'''







				A     :  Union[str, Any]   =				query.shape[-3:]

				def chunk_scanner(lowerCamelCase_:  List[str]				,	lowerCamelCase_:  int		):
								# julienne query array
								A     :  str   =				jax.lax.dynamic_slice(
								    operand=lowerCamelCase_				,	start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]				,	slice_sizes=list(query.shape[:-3]		) + [min(lowerCamelCase_				,	lowerCamelCase_		), num_heads, q_features]				,	)

								return (
								    chunk_idx + query_chunk_size,  # unused ignore it
								    _query_chunk_attention(
								        query=lowerCamelCase_				,	key=lowerCamelCase_				,	value=lowerCamelCase_				,	precision=lowerCamelCase_				,	key_chunk_size=lowerCamelCase_		),
								)

				A     :  str   =				jax.lax.scan(
				    f=lowerCamelCase_				,	init=0				,	xs=lowerCamelCase_				,	length=math.ceil(num_q / query_chunk_size		)		)

				return jnp.concatenate(lowerCamelCase_				,	axis=-3		)  # fuse the chunked result back

class     _SCREAMING_SNAKE_CASE							(		nn.Module							):
				lowerCamelCase_       =     4_2
				lowerCamelCase_       =     8
				lowerCamelCase_       =     6_4
				lowerCamelCase_       =     0.0
				lowerCamelCase_       =     False
				lowerCamelCase_       =     jnp.floataa





				def        _UpperCAmelCase						(		self :		Any							):


								"""simple docstring"""

								A     :  Union[str, Any]   =				self.dim_head * self.heads
								A     :  Dict   =				self.dim_head**-0.5

								# Weights were exported with old names {to_q, to_k, to_v, to_out}
								A     :  List[Any]   =				nn.Dense(snake_case_					,      use_bias=snake_case_					,      dtype=self.dtype					,      name='''to_q'''							)
								A     :  List[str]   =				nn.Dense(snake_case_					,      use_bias=snake_case_					,      dtype=self.dtype					,      name='''to_k'''							)
								A     :  Optional[Any]   =				nn.Dense(snake_case_					,      use_bias=snake_case_					,      dtype=self.dtype					,      name='''to_v'''							)

								A     :  Any   =				nn.Dense(self.query_dim					,      dtype=self.dtype					,      name='''to_out_0'''							)
								A     :  int   =				nn.Dropout(rate=self.dropout							)





				def        _UpperCAmelCase						(		self :		str					,      snake_case_ :		Union[str, Any]							):


								"""simple docstring"""

								A     :  str   =				tensor.shape
								A     :  int   =				self.heads
								A     :  Tuple   =				tensor.reshape(snake_case_					,      snake_case_					,      snake_case_					,      dim // head_size							)
								A     :  Optional[Any]   =				jnp.transpose(snake_case_					,      (0, 2, 1, 3)							)
								A     :  str   =				tensor.reshape(batch_size * head_size					,      snake_case_					,      dim // head_size							)
								return tensor





				def        _UpperCAmelCase						(		self :		Any					,      snake_case_ :		List[Any]							):


								"""simple docstring"""

								A     :  Tuple   =				tensor.shape
								A     :  List[str]   =				self.heads
								A     :  int   =				tensor.reshape(batch_size // head_size					,      snake_case_					,      snake_case_					,      snake_case_							)
								A     :  List[str]   =				jnp.transpose(snake_case_					,      (0, 2, 1, 3)							)
								A     :  int   =				tensor.reshape(batch_size // head_size					,      snake_case_					,      dim * head_size							)
								return tensor






				def __call__(		self :		Any					,      snake_case_ :		Tuple					,      snake_case_ :		Union[str, Any]=None					,      snake_case_ :		Dict=True							):


								"""simple docstring"""

								A     :  List[str]   =				hidden_states if context is None else context

								A     :  Tuple   =				self.query(snake_case_							)
								A     :  int   =				self.key(snake_case_							)
								A     :  Optional[int]   =				self.value(snake_case_							)

								A     :  Union[str, Any]   =				self.reshape_heads_to_batch_dim(snake_case_							)
								A     :  Dict   =				self.reshape_heads_to_batch_dim(snake_case_							)
								A     :  str   =				self.reshape_heads_to_batch_dim(snake_case_							)

								if self.use_memory_efficient_attention:
												A     :  Union[str, Any]   =				query_states.transpose(1					,      0					,      2							)
												A     :  Dict   =				key_states.transpose(1					,      0					,      2							)
												A     :  Optional[int]   =				value_states.transpose(1					,      0					,      2							)

												# this if statement create a chunk size for each layer of the unet
												# the chunk size is equal to the query_length dimension of the deepest layer of the unet

												A     :  List[Any]   =				query_states.shape[-3]
												if flatten_latent_dim % 64 == 0:
																A     :  str   =				int(flatten_latent_dim / 64							)
												elif flatten_latent_dim % 16 == 0:
																A     :  Tuple   =				int(flatten_latent_dim / 16							)
												elif flatten_latent_dim % 4 == 0:
																A     :  Optional[Any]   =				int(flatten_latent_dim / 4							)
												else:
																A     :  Tuple   =				int(snake_case_							)

												A     :  int   =				jax_memory_efficient_attention(
												    snake_case_					,      snake_case_					,      snake_case_					,      query_chunk_size=snake_case_					,      key_chunk_size=4096 * 4							)

												A     :  Optional[Any]   =				hidden_states.transpose(1					,      0					,      2							)
								else:
												# compute attentions
												A     :  List[str]   =				jnp.einsum('''b i d, b j d->b i j'''					,      snake_case_					,      snake_case_							)
												A     :  str   =				attention_scores * self.scale
												A     :  Any   =				nn.softmax(snake_case_					,      axis=2							)

												# attend to values
												A     :  Optional[Any]   =				jnp.einsum('''b i j, b j d -> b i d'''					,      snake_case_					,      snake_case_							)

								A     :  Dict   =				self.reshape_batch_dim_to_heads(snake_case_							)
								A     :  Optional[Any]   =				self.proj_attn(snake_case_							)
								return self.dropout_layer(snake_case_					,      deterministic=snake_case_							)



class     _SCREAMING_SNAKE_CASE							(		nn.Module							):
				lowerCamelCase_       =     4_2
				lowerCamelCase_       =     4_2
				lowerCamelCase_       =     4_2
				lowerCamelCase_       =     0.0
				lowerCamelCase_       =     False
				lowerCamelCase_       =     jnp.floataa
				lowerCamelCase_       =     False





				def        _UpperCAmelCase						(		self :		Dict							):


								"""simple docstring"""

								A     :  Tuple   =				FlaxAttention(
								    self.dim					,      self.n_heads					,      self.d_head					,      self.dropout					,      self.use_memory_efficient_attention					,      dtype=self.dtype							)
								# cross attention
								A     :  List[Any]   =				FlaxAttention(
								    self.dim					,      self.n_heads					,      self.d_head					,      self.dropout					,      self.use_memory_efficient_attention					,      dtype=self.dtype							)
								A     :  Optional[int]   =				FlaxFeedForward(dim=self.dim					,      dropout=self.dropout					,      dtype=self.dtype							)
								A     :  Dict   =				nn.LayerNorm(epsilon=1E-5					,      dtype=self.dtype							)
								A     :  Optional[int]   =				nn.LayerNorm(epsilon=1E-5					,      dtype=self.dtype							)
								A     :  Union[str, Any]   =				nn.LayerNorm(epsilon=1E-5					,      dtype=self.dtype							)
								A     :  Optional[Any]   =				nn.Dropout(rate=self.dropout							)






				def __call__(		self :		Dict					,      snake_case_ :		int					,      snake_case_ :		str					,      snake_case_ :		Any=True							):


								"""simple docstring"""

								A     :  Dict   =				hidden_states
								if self.only_cross_attention:
												A     :  str   =				self.attna(self.norma(snake_case_							)					,      snake_case_					,      deterministic=snake_case_							)
								else:
												A     :  Any   =				self.attna(self.norma(snake_case_							)					,      deterministic=snake_case_							)
								A     :  Optional[Any]   =				hidden_states + residual

								# cross attention
								A     :  Optional[int]   =				hidden_states
								A     :  Optional[Any]   =				self.attna(self.norma(snake_case_							)					,      snake_case_					,      deterministic=snake_case_							)
								A     :  Tuple   =				hidden_states + residual

								# feed forward
								A     :  str   =				hidden_states
								A     :  Optional[int]   =				self.ff(self.norma(snake_case_							)					,      deterministic=snake_case_							)
								A     :  Optional[int]   =				hidden_states + residual

								return self.dropout_layer(snake_case_					,      deterministic=snake_case_							)



class     _SCREAMING_SNAKE_CASE							(		nn.Module							):
				lowerCamelCase_       =     4_2
				lowerCamelCase_       =     4_2
				lowerCamelCase_       =     4_2
				lowerCamelCase_       =     1
				lowerCamelCase_       =     0.0
				lowerCamelCase_       =     False
				lowerCamelCase_       =     False
				lowerCamelCase_       =     jnp.floataa
				lowerCamelCase_       =     False





				def        _UpperCAmelCase						(		self :		Union[str, Any]							):


								"""simple docstring"""

								A     :  List[Any]   =				nn.GroupNorm(num_groups=32					,      epsilon=1E-5							)

								A     :  List[str]   =				self.n_heads * self.d_head
								if self.use_linear_projection:
												A     :  Dict   =				nn.Dense(snake_case_					,      dtype=self.dtype							)
								else:
												A     :  Tuple   =				nn.Conv(
												    snake_case_					,      kernel_size=(1, 1)					,      strides=(1, 1)					,      padding='''VALID'''					,      dtype=self.dtype					,      )

								A     :  Dict   =				[
								    FlaxBasicTransformerBlock(
								        snake_case_					,      self.n_heads					,      self.d_head					,      dropout=self.dropout					,      only_cross_attention=self.only_cross_attention					,      dtype=self.dtype					,      use_memory_efficient_attention=self.use_memory_efficient_attention					,      )
								    for _ in range(self.depth							)
								]

								if self.use_linear_projection:
												A     :  int   =				nn.Dense(snake_case_					,      dtype=self.dtype							)
								else:
												A     :  List[Any]   =				nn.Conv(
												    snake_case_					,      kernel_size=(1, 1)					,      strides=(1, 1)					,      padding='''VALID'''					,      dtype=self.dtype					,      )

								A     :  List[Any]   =				nn.Dropout(rate=self.dropout							)






				def __call__(		self :		Dict					,      snake_case_ :		List[Any]					,      snake_case_ :		Union[str, Any]					,      snake_case_ :		str=True							):


								"""simple docstring"""

								A     :  Union[str, Any]   =				hidden_states.shape
								A     :  Optional[int]   =				hidden_states
								A     :  int   =				self.norm(snake_case_							)
								if self.use_linear_projection:
												A     :  Any   =				hidden_states.reshape(snake_case_					,      height * width					,      snake_case_							)
												A     :  Union[str, Any]   =				self.proj_in(snake_case_							)
								else:
												A     :  List[Any]   =				self.proj_in(snake_case_							)
												A     :  List[Any]   =				hidden_states.reshape(snake_case_					,      height * width					,      snake_case_							)

								for transformer_block in self.transformer_blocks:
												A     :  Union[str, Any]   =				transformer_block(snake_case_					,      snake_case_					,      deterministic=snake_case_							)

								if self.use_linear_projection:
												A     :  List[Any]   =				self.proj_out(snake_case_							)
												A     :  Dict   =				hidden_states.reshape(snake_case_					,      snake_case_					,      snake_case_					,      snake_case_							)
								else:
												A     :  List[str]   =				hidden_states.reshape(snake_case_					,      snake_case_					,      snake_case_					,      snake_case_							)
												A     :  Optional[int]   =				self.proj_out(snake_case_							)

								A     :  Union[str, Any]   =				hidden_states + residual
								return self.dropout_layer(snake_case_					,      deterministic=snake_case_							)



class     _SCREAMING_SNAKE_CASE							(		nn.Module							):
				lowerCamelCase_       =     4_2
				lowerCamelCase_       =     0.0
				lowerCamelCase_       =     jnp.floataa





				def        _UpperCAmelCase						(		self :		Any							):


								"""simple docstring"""

								A     :  Optional[int]   =				FlaxGEGLU(self.dim					,      self.dropout					,      self.dtype							)
								A     :  Tuple   =				nn.Dense(self.dim					,      dtype=self.dtype							)






				def __call__(		self :		List[Any]					,      snake_case_ :		List[Any]					,      snake_case_ :		List[str]=True							):


								"""simple docstring"""

								A     :  Dict   =				self.net_a(snake_case_					,      deterministic=snake_case_							)
								A     :  Union[str, Any]   =				self.net_a(snake_case_							)
								return hidden_states



class     _SCREAMING_SNAKE_CASE							(		nn.Module							):
				lowerCamelCase_       =     4_2
				lowerCamelCase_       =     0.0
				lowerCamelCase_       =     jnp.floataa





				def        _UpperCAmelCase						(		self :		List[str]							):


								"""simple docstring"""

								A     :  List[Any]   =				self.dim * 4
								A     :  Optional[Any]   =				nn.Dense(inner_dim * 2					,      dtype=self.dtype							)
								A     :  Dict   =				nn.Dropout(rate=self.dropout							)






				def __call__(		self :		Tuple					,      snake_case_ :		Optional[Any]					,      snake_case_ :		Optional[int]=True							):


								"""simple docstring"""

								A     :  Optional[int]   =				self.proj(snake_case_							)
								A     :  str   =				jnp.split(snake_case_					,      2					,      axis=2							)
								return self.dropout_layer(hidden_linear * nn.gelu(snake_case_							)					,      deterministic=snake_case_							)

UpperCamelCase_  =	{
    "0": 1,
    "1": 1,
    "2": 2,
    "3": 6,
    "4": 24,
    "5": 1_20,
    "6": 7_20,
    "7": 50_40,
    "8": 4_03_20,
    "9": 36_28_80,
}

UpperCamelCase_  =	{1_45: 1_45}

UpperCamelCase_  =	{
    1_45: 0,
    1_69: 3,
    3_63_01: 3,
    14_54: 3,
    8_71: 2,
    4_53_61: 2,
    8_72: 2,
}





def 							_lowerCamelCase			(					lowerCamelCase_:  int		):



				'''simple docstring'''







				if n in CACHE_SUM_DIGIT_FACTORIALS:
								return CACHE_SUM_DIGIT_FACTORIALS[n]
				A     :  Tuple   =				sum(DIGIT_FACTORIALS[let] for let in str(lowerCamelCase_		)		)
				A     :  Dict   =				ret
				return ret





def 							_lowerCamelCase			(					lowerCamelCase_:  int				,	lowerCamelCase_:  set | None = None		):



				'''simple docstring'''







				A     :  Optional[int]   =				previous or set()
				if n in CHAIN_LENGTH_CACHE:
								return CHAIN_LENGTH_CACHE[n]
				A     :  Dict   =				sum_digit_factorials(lowerCamelCase_		)
				if next_number in previous:
								A     :  Optional[int]   =				0
								return 0
				else:
								previous.add(lowerCamelCase_		)
								A     :  str   =				1 + chain_length(lowerCamelCase_				,	lowerCamelCase_		)
								A     :  Any   =				ret
								return ret





def 							_lowerCamelCase			(					lowerCamelCase_:  int = 60				,	lowerCamelCase_:  int = 100_0000		):



				'''simple docstring'''







				return sum(1 for i in range(1				,	lowerCamelCase_		) if chain_length(lowerCamelCase_		) == num_terms		)


if __name__ == "__main__":
					print(F'''{solution() = }''')