


'''simple docstring'''




import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp





def 							a_       (						lowercase							,						lowercase							,						lowercase							,						lowercase							,						lowercase = 4096    )			->Dict:






				'''simple docstring'''

				__lowerCamelCase		:      Any											=					key.shape[-3:]
				__lowerCamelCase		:      List[Any]											=					value.shape[-1]
				__lowerCamelCase		:      Dict											=					min(lowercase							,						lowercase    )
				__lowerCamelCase		:      int											=					query / jnp.sqrt(lowercase    )

				@functools.partial(jax.checkpoint							,						prevent_cse=lowercase    )
				def summarize_chunk(lowercase							,						lowercase							,						lowercase    ):
								__lowerCamelCase		:      List[str]											=					jnp.einsum("...qhd,...khd->...qhk"							,						lowercase							,						lowercase							,						precision=lowercase    )

								__lowerCamelCase		:      Union[str, Any]											=					jnp.max(lowercase							,						axis=-1							,						keepdims=lowercase    )
								__lowerCamelCase		:      Dict											=					jax.lax.stop_gradient(lowercase    )
								__lowerCamelCase		:      Optional[int]											=					jnp.exp(attn_weights - max_score    )

								__lowerCamelCase		:      Tuple											=					jnp.einsum("...vhf,...qhv->...qhf"							,						lowercase							,						lowercase							,						precision=lowercase    )
								__lowerCamelCase		:      List[Any]											=					jnp.einsum("...qhk->...qh"							,						lowercase    )

								return (exp_values, exp_weights.sum(axis=-1    ), max_score)

				def chunk_scanner(lowercase    ):
								# julienne key array
								__lowerCamelCase		:      Tuple											=					jax.lax.dynamic_slice(
								    operand=lowercase							,						start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]							,						slice_sizes=list(key.shape[:-3]    ) + [key_chunk_size, num_heads, k_features]							,						)

								# julienne value array
								__lowerCamelCase		:      List[str]											=					jax.lax.dynamic_slice(
								    operand=lowercase							,						start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]							,						slice_sizes=list(value.shape[:-3]    ) + [key_chunk_size, num_heads, v_features]							,						)

								return summarize_chunk(lowercase							,						lowercase							,						lowercase    )

				__lowerCamelCase		:      List[str]											=					jax.lax.map(f=lowercase							,						xs=jnp.arange(0							,						lowercase							,						lowercase    )    )

				__lowerCamelCase		:      Tuple											=					jnp.max(lowercase							,						axis=0							,						keepdims=lowercase    )
				__lowerCamelCase		:      Tuple											=					jnp.exp(chunk_max - global_max    )

				chunk_values *= jnp.expand_dims(lowercase							,						axis=-1    )
				chunk_weights *= max_diffs

				__lowerCamelCase		:      Dict											=					chunk_values.sum(axis=0    )
				__lowerCamelCase		:      Union[str, Any]											=					jnp.expand_dims(lowercase							,						-1    ).sum(axis=0    )

				return all_values / all_weights





def 							a_       (						lowercase							,						lowercase							,						lowercase							,						lowercase=jax.lax.Precision.HIGHEST							,						lowercase = 1024							,						lowercase = 4096    )			->int:






				'''simple docstring'''

				__lowerCamelCase		:      str											=					query.shape[-3:]

				def chunk_scanner(lowercase							,						lowercase    ):
								# julienne query array
								__lowerCamelCase		:      List[Any]											=					jax.lax.dynamic_slice(
								    operand=lowercase							,						start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]							,						slice_sizes=list(query.shape[:-3]    ) + [min(lowercase							,						lowercase    ), num_heads, q_features]							,						)

								return (
								    chunk_idx + query_chunk_size,  # unused ignore it
								    _query_chunk_attention(
								        query=lowercase							,						key=lowercase							,						value=lowercase							,						precision=lowercase							,						key_chunk_size=lowercase    ),
								)

				__lowerCamelCase		:      Optional[Any]											=					jax.lax.scan(
				    f=lowercase							,						init=0							,						xs=lowercase							,						length=math.ceil(num_q / query_chunk_size    )    )

				return jnp.concatenate(lowercase							,						axis=-3    )  # fuse the chunked result back





class _snake_case   ( nn.Module   ):
							__a												=     4_2
							__a												=     8
							__a												=     6_4
							__a												=     0.0
							__a												=     False
							__a												=     jnp.floataa







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->List[str]:



											"""simple docstring"""







											__lowerCamelCase		:      int											=					self.dim_head * self.heads
											__lowerCamelCase		:      List[str]											=					self.dim_head**-0.5

											# Weights were exported with old names {to_q, to_k, to_v, to_out}
											__lowerCamelCase		:      Optional[Any]											=					nn.Dense(__A			,				use_bias=__A			,				dtype=self.dtype			,				name="to_q"	)
											__lowerCamelCase		:      Optional[Any]											=					nn.Dense(__A			,				use_bias=__A			,				dtype=self.dtype			,				name="to_k"	)
											__lowerCamelCase		:      List[str]											=					nn.Dense(__A			,				use_bias=__A			,				dtype=self.dtype			,				name="to_v"	)

											__lowerCamelCase		:      List[Any]											=					nn.Dense(self.query_dim			,				dtype=self.dtype			,				name="to_out_0"	)
											__lowerCamelCase		:      Dict											=					nn.Dropout(rate=self.dropout	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->Optional[Any]:



											"""simple docstring"""







											__lowerCamelCase		:      Optional[int]											=					tensor.shape
											__lowerCamelCase		:      List[str]											=					self.heads
											__lowerCamelCase		:      List[Any]											=					tensor.reshape(__A			,				__A			,				__A			,				dim // head_size	)
											__lowerCamelCase		:      str											=					jnp.transpose(__A			,				(0, 2, 1, 3)	)
											__lowerCamelCase		:      Optional[int]											=					tensor.reshape(batch_size * head_size			,				__A			,				dim // head_size	)
											return tensor







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->str:



											"""simple docstring"""







											__lowerCamelCase		:      int											=					tensor.shape
											__lowerCamelCase		:      Dict											=					self.heads
											__lowerCamelCase		:      List[str]											=					tensor.reshape(batch_size // head_size			,				__A			,				__A			,				__A	)
											__lowerCamelCase		:      int											=					jnp.transpose(__A			,				(0, 2, 1, 3)	)
											__lowerCamelCase		:      List[Any]											=					tensor.reshape(batch_size // head_size			,				__A			,				dim * head_size	)
											return tensor







							def __call__(   self			,				__A			,				__A=None			,				__A=True	)     ->List[str]:



											"""simple docstring"""







											__lowerCamelCase		:      List[str]											=					hidden_states if context is None else context

											__lowerCamelCase		:      Optional[int]											=					self.query(__A	)
											__lowerCamelCase		:      Dict											=					self.key(__A	)
											__lowerCamelCase		:      Optional[int]											=					self.value(__A	)

											__lowerCamelCase		:      str											=					self.reshape_heads_to_batch_dim(__A	)
											__lowerCamelCase		:      List[Any]											=					self.reshape_heads_to_batch_dim(__A	)
											__lowerCamelCase		:      Tuple											=					self.reshape_heads_to_batch_dim(__A	)

											if self.use_memory_efficient_attention:
															__lowerCamelCase		:      str											=					query_states.transpose(1			,				0			,				2	)
															__lowerCamelCase		:      Dict											=					key_states.transpose(1			,				0			,				2	)
															__lowerCamelCase		:      Dict											=					value_states.transpose(1			,				0			,				2	)

															# this if statement create a chunk size for each layer of the unet
															# the chunk size is equal to the query_length dimension of the deepest layer of the unet

															__lowerCamelCase		:      Optional[Any]											=					query_states.shape[-3]
															if flatten_latent_dim % 64 == 0:
																			__lowerCamelCase		:      Optional[int]											=					int(flatten_latent_dim / 64	)
															elif flatten_latent_dim % 16 == 0:
																			__lowerCamelCase		:      Tuple											=					int(flatten_latent_dim / 16	)
															elif flatten_latent_dim % 4 == 0:
																			__lowerCamelCase		:      int											=					int(flatten_latent_dim / 4	)
															else:
																			__lowerCamelCase		:      int											=					int(__A	)

															__lowerCamelCase		:      Union[str, Any]											=					jax_memory_efficient_attention(
															    __A			,				__A			,				__A			,				query_chunk_size=__A			,				key_chunk_size=4096 * 4	)

															__lowerCamelCase		:      Dict											=					hidden_states.transpose(1			,				0			,				2	)
											else:
															# compute attentions
															__lowerCamelCase		:      Union[str, Any]											=					jnp.einsum("b i d, b j d->b i j"			,				__A			,				__A	)
															__lowerCamelCase		:      Tuple											=					attention_scores * self.scale
															__lowerCamelCase		:      Optional[int]											=					nn.softmax(__A			,				axis=2	)

															# attend to values
															__lowerCamelCase		:      List[str]											=					jnp.einsum("b i j, b j d -> b i d"			,				__A			,				__A	)

											__lowerCamelCase		:      str											=					self.reshape_batch_dim_to_heads(__A	)
											__lowerCamelCase		:      str											=					self.proj_attn(__A	)
											return self.dropout_layer(__A			,				deterministic=__A	)





class _snake_case   ( nn.Module   ):
							__a												=     4_2
							__a												=     4_2
							__a												=     4_2
							__a												=     0.0
							__a												=     False
							__a												=     jnp.floataa
							__a												=     False







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Union[str, Any]:



											"""simple docstring"""







											__lowerCamelCase		:      str											=					FlaxAttention(
											    self.dim			,				self.n_heads			,				self.d_head			,				self.dropout			,				self.use_memory_efficient_attention			,				dtype=self.dtype	)
											# cross attention
											__lowerCamelCase		:      str											=					FlaxAttention(
											    self.dim			,				self.n_heads			,				self.d_head			,				self.dropout			,				self.use_memory_efficient_attention			,				dtype=self.dtype	)
											__lowerCamelCase		:      Union[str, Any]											=					FlaxFeedForward(dim=self.dim			,				dropout=self.dropout			,				dtype=self.dtype	)
											__lowerCamelCase		:      List[Any]											=					nn.LayerNorm(epsilon=1E-5			,				dtype=self.dtype	)
											__lowerCamelCase		:      Tuple											=					nn.LayerNorm(epsilon=1E-5			,				dtype=self.dtype	)
											__lowerCamelCase		:      Tuple											=					nn.LayerNorm(epsilon=1E-5			,				dtype=self.dtype	)
											__lowerCamelCase		:      int											=					nn.Dropout(rate=self.dropout	)







							def __call__(   self			,				__A			,				__A			,				__A=True	)     ->int:



											"""simple docstring"""







											__lowerCamelCase		:      Dict											=					hidden_states
											if self.only_cross_attention:
															__lowerCamelCase		:      Tuple											=					self.attna(self.norma(__A	)			,				__A			,				deterministic=__A	)
											else:
															__lowerCamelCase		:      Optional[Any]											=					self.attna(self.norma(__A	)			,				deterministic=__A	)
											__lowerCamelCase		:      List[Any]											=					hidden_states + residual

											# cross attention
											__lowerCamelCase		:      Tuple											=					hidden_states
											__lowerCamelCase		:      Optional[int]											=					self.attna(self.norma(__A	)			,				__A			,				deterministic=__A	)
											__lowerCamelCase		:      str											=					hidden_states + residual

											# feed forward
											__lowerCamelCase		:      Optional[int]											=					hidden_states
											__lowerCamelCase		:      List[str]											=					self.ff(self.norma(__A	)			,				deterministic=__A	)
											__lowerCamelCase		:      Tuple											=					hidden_states + residual

											return self.dropout_layer(__A			,				deterministic=__A	)





class _snake_case   ( nn.Module   ):
							__a												=     4_2
							__a												=     4_2
							__a												=     4_2
							__a												=     1
							__a												=     0.0
							__a												=     False
							__a												=     False
							__a												=     jnp.floataa
							__a												=     False







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->str:



											"""simple docstring"""







											__lowerCamelCase		:      List[Any]											=					nn.GroupNorm(num_groups=32			,				epsilon=1E-5	)

											__lowerCamelCase		:      List[str]											=					self.n_heads * self.d_head
											if self.use_linear_projection:
															__lowerCamelCase		:      Union[str, Any]											=					nn.Dense(__A			,				dtype=self.dtype	)
											else:
															__lowerCamelCase		:      Optional[Any]											=					nn.Conv(
															    __A			,				kernel_size=(1, 1)			,				strides=(1, 1)			,				padding="VALID"			,				dtype=self.dtype			,				)

											__lowerCamelCase		:      Tuple											=					[
											    FlaxBasicTransformerBlock(
											        __A			,				self.n_heads			,				self.d_head			,				dropout=self.dropout			,				only_cross_attention=self.only_cross_attention			,				dtype=self.dtype			,				use_memory_efficient_attention=self.use_memory_efficient_attention			,				)
											    for _ in range(self.depth	)
											]

											if self.use_linear_projection:
															__lowerCamelCase		:      Any											=					nn.Dense(__A			,				dtype=self.dtype	)
											else:
															__lowerCamelCase		:      Optional[int]											=					nn.Conv(
															    __A			,				kernel_size=(1, 1)			,				strides=(1, 1)			,				padding="VALID"			,				dtype=self.dtype			,				)

											__lowerCamelCase		:      Dict											=					nn.Dropout(rate=self.dropout	)







							def __call__(   self			,				__A			,				__A			,				__A=True	)     ->Optional[Any]:



											"""simple docstring"""







											__lowerCamelCase		:      Optional[Any]											=					hidden_states.shape
											__lowerCamelCase		:      Dict											=					hidden_states
											__lowerCamelCase		:      int											=					self.norm(__A	)
											if self.use_linear_projection:
															__lowerCamelCase		:      Tuple											=					hidden_states.reshape(__A			,				height * width			,				__A	)
															__lowerCamelCase		:      int											=					self.proj_in(__A	)
											else:
															__lowerCamelCase		:      Dict											=					self.proj_in(__A	)
															__lowerCamelCase		:      Optional[int]											=					hidden_states.reshape(__A			,				height * width			,				__A	)

											for transformer_block in self.transformer_blocks:
															__lowerCamelCase		:      Dict											=					transformer_block(__A			,				__A			,				deterministic=__A	)

											if self.use_linear_projection:
															__lowerCamelCase		:      List[Any]											=					self.proj_out(__A	)
															__lowerCamelCase		:      Any											=					hidden_states.reshape(__A			,				__A			,				__A			,				__A	)
											else:
															__lowerCamelCase		:      Dict											=					hidden_states.reshape(__A			,				__A			,				__A			,				__A	)
															__lowerCamelCase		:      List[str]											=					self.proj_out(__A	)

											__lowerCamelCase		:      Any											=					hidden_states + residual
											return self.dropout_layer(__A			,				deterministic=__A	)





class _snake_case   ( nn.Module   ):
							__a												=     4_2
							__a												=     0.0
							__a												=     jnp.floataa







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:



											"""simple docstring"""







											__lowerCamelCase		:      Union[str, Any]											=					FlaxGEGLU(self.dim			,				self.dropout			,				self.dtype	)
											__lowerCamelCase		:      Any											=					nn.Dense(self.dim			,				dtype=self.dtype	)







							def __call__(   self			,				__A			,				__A=True	)     ->Tuple:



											"""simple docstring"""







											__lowerCamelCase		:      str											=					self.net_a(__A			,				deterministic=__A	)
											__lowerCamelCase		:      Union[str, Any]											=					self.net_a(__A	)
											return hidden_states





class _snake_case   ( nn.Module   ):
							__a												=     4_2
							__a												=     0.0
							__a												=     jnp.floataa







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->List[Any]:



											"""simple docstring"""







											__lowerCamelCase		:      List[str]											=					self.dim * 4
											__lowerCamelCase		:      Dict											=					nn.Dense(inner_dim * 2			,				dtype=self.dtype	)
											__lowerCamelCase		:      List[str]											=					nn.Dropout(rate=self.dropout	)







							def __call__(   self			,				__A			,				__A=True	)     ->Optional[int]:



											"""simple docstring"""







											__lowerCamelCase		:      Any											=					self.proj(__A	)
											__lowerCamelCase		:      List[str]											=					jnp.split(__A			,				2			,				axis=2	)
											return self.dropout_layer(hidden_linear * nn.gelu(__A	)			,				deterministic=__A	)









'''simple docstring'''




def 							a_       (						lowercase    )			->str:






				'''simple docstring'''

				return " ".join(sorted(set(sentence.split()    )    )    )


if __name__ == "__main__":
				import doctest

				doctest.testmod()





