






import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp




def   lowercase__   (  __snake_case					:							Any  , __snake_case					:							Optional[int]  , __snake_case					:							Optional[int]  , __snake_case					:							List[Any]  , __snake_case					:							Dict = 4_096						):


					'''simple docstring'''


					UpperCAmelCase_					:   Any   =		key.shape[-3:]
					UpperCAmelCase_					:   List[Any]   =		value.shape[-1]
					UpperCAmelCase_					:   Dict   =		min(__snake_case  , __snake_case						)
					UpperCAmelCase_					:   int   =		query / jnp.sqrt(__snake_case						)

					@functools.partial(jax.checkpoint  , prevent_cse=__snake_case						)
					def summarize_chunk(__snake_case					:							List[str]  , __snake_case					:							List[Any]  , __snake_case					:							int						):
										UpperCAmelCase_					:   List[str]   =		jnp.einsum('...qhd,...khd->...qhk'  , __snake_case  , __snake_case  , precision=__snake_case						)

										UpperCAmelCase_					:   Union[str, Any]   =		jnp.max(__snake_case  , axis=-1  , keepdims=__snake_case						)
										UpperCAmelCase_					:   Dict   =		jax.lax.stop_gradient(__snake_case						)
										UpperCAmelCase_					:   Optional[int]   =		jnp.exp(attn_weights - max_score						)

										UpperCAmelCase_					:   Tuple   =		jnp.einsum('...vhf,...qhv->...qhf'  , __snake_case  , __snake_case  , precision=__snake_case						)
										UpperCAmelCase_					:   List[Any]   =		jnp.einsum('...qhk->...qh'  , __snake_case						)

										return (exp_values, exp_weights.sum(axis=-1						), max_score)

					def chunk_scanner(__snake_case					:							List[str]						):
										# julienne key array
										UpperCAmelCase_					:   Tuple   =		jax.lax.dynamic_slice(
										    operand=__snake_case  , start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]  , slice_sizes=list(key.shape[:-3]						) + [key_chunk_size, num_heads, k_features]  , )

										# julienne value array
										UpperCAmelCase_					:   List[str]   =		jax.lax.dynamic_slice(
										    operand=__snake_case  , start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]  , slice_sizes=list(value.shape[:-3]						) + [key_chunk_size, num_heads, v_features]  , )

										return summarize_chunk(__snake_case  , __snake_case  , __snake_case						)

					UpperCAmelCase_					:   List[str]   =		jax.lax.map(f=__snake_case  , xs=jnp.arange(0  , __snake_case  , __snake_case						)						)

					UpperCAmelCase_					:   Tuple   =		jnp.max(__snake_case  , axis=0  , keepdims=__snake_case						)
					UpperCAmelCase_					:   Tuple   =		jnp.exp(chunk_max - global_max						)

					chunk_values *= jnp.expand_dims(__snake_case  , axis=-1						)
					chunk_weights *= max_diffs

					UpperCAmelCase_					:   Dict   =		chunk_values.sum(axis=0						)
					UpperCAmelCase_					:   Union[str, Any]   =		jnp.expand_dims(__snake_case  , -1						).sum(axis=0						)

					return all_values / all_weights




def   lowercase__   (  __snake_case					:							List[str]  , __snake_case					:							int  , __snake_case					:							List[Any]  , __snake_case					:							str=jax.lax.Precision.HIGHEST  , __snake_case					:							str = 1_024  , __snake_case					:							Optional[int] = 4_096						):


					'''simple docstring'''


					UpperCAmelCase_					:   str   =		query.shape[-3:]

					def chunk_scanner(__snake_case					:							Union[str, Any]  , __snake_case					:							Optional[Any]						):
										# julienne query array
										UpperCAmelCase_					:   List[Any]   =		jax.lax.dynamic_slice(
										    operand=__snake_case  , start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]  , slice_sizes=list(query.shape[:-3]						) + [min(__snake_case  , __snake_case						), num_heads, q_features]  , )

										return (
										    chunk_idx + query_chunk_size,  # unused ignore it
										    _query_chunk_attention(
										        query=__snake_case  , key=__snake_case  , value=__snake_case  , precision=__snake_case  , key_chunk_size=__snake_case						),
										)

					UpperCAmelCase_					:   Optional[Any]   =		jax.lax.scan(
					    f=__snake_case  , init=0  , xs=__snake_case  , length=math.ceil(num_q / query_chunk_size						)						)

					return jnp.concatenate(__snake_case  , axis=-3						)  # fuse the chunked result back



class 						lowerCamelCase   (nn.Module    ):







		'''simple docstring'''



		_snake_case					:					Any 		=						4_2
		_snake_case					:					str 		=						8
		_snake_case					:					Dict 		=						6_4
		_snake_case					:					List[str] 		=						0.0
		_snake_case					:					List[Any] 		=						False
		_snake_case					:					str 		=						jnp.floataa



		def       __UpperCAmelCase    (							self     )			->							List[str]:
							UpperCAmelCase_					:   int   =		self.dim_head * self.heads
							UpperCAmelCase_					:   List[str]   =		self.dim_head**-0.5

							# Weights were exported with old names {to_q, to_k, to_v, to_out}
							UpperCAmelCase_					:   Optional[Any]   =		nn.Dense(__A       ,	use_bias=__A       ,	dtype=self.dtype       ,	name='to_q'     )
							UpperCAmelCase_					:   Optional[Any]   =		nn.Dense(__A       ,	use_bias=__A       ,	dtype=self.dtype       ,	name='to_k'     )
							UpperCAmelCase_					:   List[str]   =		nn.Dense(__A       ,	use_bias=__A       ,	dtype=self.dtype       ,	name='to_v'     )

							UpperCAmelCase_					:   List[Any]   =		nn.Dense(self.query_dim       ,	dtype=self.dtype       ,	name='to_out_0'     )
							UpperCAmelCase_					:   Dict   =		nn.Dropout(rate=self.dropout     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase     )			->							Optional[Any]:
							UpperCAmelCase_					:   Optional[int]   =		tensor.shape
							UpperCAmelCase_					:   List[str]   =		self.heads
							UpperCAmelCase_					:   List[Any]   =		tensor.reshape(__A       ,	__A       ,	__A       ,	dim // head_size     )
							UpperCAmelCase_					:   str   =		jnp.transpose(__A       ,	(0, 2, 1, 3)     )
							UpperCAmelCase_					:   Optional[int]   =		tensor.reshape(batch_size * head_size       ,	__A       ,	dim // head_size     )
							return tensor



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase     )			->							str:
							UpperCAmelCase_					:   int   =		tensor.shape
							UpperCAmelCase_					:   Dict   =		self.heads
							UpperCAmelCase_					:   List[str]   =		tensor.reshape(batch_size // head_size       ,	__A       ,	__A       ,	__A     )
							UpperCAmelCase_					:   int   =		jnp.transpose(__A       ,	(0, 2, 1, 3)     )
							UpperCAmelCase_					:   List[Any]   =		tensor.reshape(batch_size // head_size       ,	__A       ,	dim * head_size     )
							return tensor



		def __call__(							self       ,	_UpperCamelCase       ,	_UpperCamelCase=None       ,	_UpperCamelCase=True     )			->							List[str]:
							UpperCAmelCase_					:   List[str]   =		hidden_states if context is None else context

							UpperCAmelCase_					:   Optional[int]   =		self.query(__A     )
							UpperCAmelCase_					:   Dict   =		self.key(__A     )
							UpperCAmelCase_					:   Optional[int]   =		self.value(__A     )

							UpperCAmelCase_					:   str   =		self.reshape_heads_to_batch_dim(__A     )
							UpperCAmelCase_					:   List[Any]   =		self.reshape_heads_to_batch_dim(__A     )
							UpperCAmelCase_					:   Tuple   =		self.reshape_heads_to_batch_dim(__A     )

							if self.use_memory_efficient_attention:
												UpperCAmelCase_					:   str   =		query_states.transpose(1       ,	0       ,	2     )
												UpperCAmelCase_					:   Dict   =		key_states.transpose(1       ,	0       ,	2     )
												UpperCAmelCase_					:   Dict   =		value_states.transpose(1       ,	0       ,	2     )

												# this if statement create a chunk size for each layer of the unet
												# the chunk size is equal to the query_length dimension of the deepest layer of the unet

												UpperCAmelCase_					:   Optional[Any]   =		query_states.shape[-3]
												if flatten_latent_dim % 6_4 == 0:
																	UpperCAmelCase_					:   Optional[int]   =		int(flatten_latent_dim / 6_4     )
												elif flatten_latent_dim % 1_6 == 0:
																	UpperCAmelCase_					:   Tuple   =		int(flatten_latent_dim / 1_6     )
												elif flatten_latent_dim % 4 == 0:
																	UpperCAmelCase_					:   int   =		int(flatten_latent_dim / 4     )
												else:
																	UpperCAmelCase_					:   int   =		int(__A     )

												UpperCAmelCase_					:   Union[str, Any]   =		jax_memory_efficient_attention(
												    __A       ,	__A       ,	__A       ,	query_chunk_size=__A       ,	key_chunk_size=4_0_9_6 * 4     )

												UpperCAmelCase_					:   Dict   =		hidden_states.transpose(1       ,	0       ,	2     )
							else:
												# compute attentions
												UpperCAmelCase_					:   Union[str, Any]   =		jnp.einsum('b i d, b j d->b i j'       ,	__A       ,	__A     )
												UpperCAmelCase_					:   Tuple   =		attention_scores * self.scale
												UpperCAmelCase_					:   Optional[int]   =		nn.softmax(__A       ,	axis=2     )

												# attend to values
												UpperCAmelCase_					:   List[str]   =		jnp.einsum('b i j, b j d -> b i d'       ,	__A       ,	__A     )

							UpperCAmelCase_					:   str   =		self.reshape_batch_dim_to_heads(__A     )
							UpperCAmelCase_					:   str   =		self.proj_attn(__A     )
							return self.dropout_layer(__A       ,	deterministic=__A     )





class 						lowerCamelCase   (nn.Module    ):







		'''simple docstring'''



		_snake_case					:					Optional[Any] 		=						4_2
		_snake_case					:					Union[str, Any] 		=						4_2
		_snake_case					:					Union[str, Any] 		=						4_2
		_snake_case					:					int 		=						0.0
		_snake_case					:					str 		=						False
		_snake_case					:					str 		=						jnp.floataa
		_snake_case					:					int 		=						False



		def       __UpperCAmelCase    (							self     )			->							Union[str, Any]:
							UpperCAmelCase_					:   str   =		FlaxAttention(
							    self.dim       ,	self.n_heads       ,	self.d_head       ,	self.dropout       ,	self.use_memory_efficient_attention       ,	dtype=self.dtype     )
							# cross attention
							UpperCAmelCase_					:   str   =		FlaxAttention(
							    self.dim       ,	self.n_heads       ,	self.d_head       ,	self.dropout       ,	self.use_memory_efficient_attention       ,	dtype=self.dtype     )
							UpperCAmelCase_					:   Union[str, Any]   =		FlaxFeedForward(dim=self.dim       ,	dropout=self.dropout       ,	dtype=self.dtype     )
							UpperCAmelCase_					:   List[Any]   =		nn.LayerNorm(epsilon=1E-5       ,	dtype=self.dtype     )
							UpperCAmelCase_					:   Tuple   =		nn.LayerNorm(epsilon=1E-5       ,	dtype=self.dtype     )
							UpperCAmelCase_					:   Tuple   =		nn.LayerNorm(epsilon=1E-5       ,	dtype=self.dtype     )
							UpperCAmelCase_					:   int   =		nn.Dropout(rate=self.dropout     )



		def __call__(							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase=True     )			->							int:
							UpperCAmelCase_					:   Dict   =		hidden_states
							if self.only_cross_attention:
												UpperCAmelCase_					:   Tuple   =		self.attna(self.norma(__A     )       ,	__A       ,	deterministic=__A     )
							else:
												UpperCAmelCase_					:   Optional[Any]   =		self.attna(self.norma(__A     )       ,	deterministic=__A     )
							UpperCAmelCase_					:   List[Any]   =		hidden_states + residual

							# cross attention
							UpperCAmelCase_					:   Tuple   =		hidden_states
							UpperCAmelCase_					:   Optional[int]   =		self.attna(self.norma(__A     )       ,	__A       ,	deterministic=__A     )
							UpperCAmelCase_					:   str   =		hidden_states + residual

							# feed forward
							UpperCAmelCase_					:   Optional[int]   =		hidden_states
							UpperCAmelCase_					:   List[str]   =		self.ff(self.norma(__A     )       ,	deterministic=__A     )
							UpperCAmelCase_					:   Tuple   =		hidden_states + residual

							return self.dropout_layer(__A       ,	deterministic=__A     )





class 						lowerCamelCase   (nn.Module    ):







		'''simple docstring'''



		_snake_case					:					Optional[Any] 		=						4_2
		_snake_case					:					List[str] 		=						4_2
		_snake_case					:					Optional[int] 		=						4_2
		_snake_case					:					Optional[int] 		=						1
		_snake_case					:					Tuple 		=						0.0
		_snake_case					:					Tuple 		=						False
		_snake_case					:					List[Any] 		=						False
		_snake_case					:					Any 		=						jnp.floataa
		_snake_case					:					List[Any] 		=						False



		def       __UpperCAmelCase    (							self     )			->							str:
							UpperCAmelCase_					:   List[Any]   =		nn.GroupNorm(num_groups=3_2       ,	epsilon=1E-5     )

							UpperCAmelCase_					:   List[str]   =		self.n_heads * self.d_head
							if self.use_linear_projection:
												UpperCAmelCase_					:   Union[str, Any]   =		nn.Dense(__A       ,	dtype=self.dtype     )
							else:
												UpperCAmelCase_					:   Optional[Any]   =		nn.Conv(
												    __A       ,	kernel_size=(1, 1)       ,	strides=(1, 1)       ,	padding='VALID'       ,	dtype=self.dtype       ,	)

							UpperCAmelCase_					:   Tuple   =		[
							    FlaxBasicTransformerBlock(
							        __A       ,	self.n_heads       ,	self.d_head       ,	dropout=self.dropout       ,	only_cross_attention=self.only_cross_attention       ,	dtype=self.dtype       ,	use_memory_efficient_attention=self.use_memory_efficient_attention       ,	)
							    for _ in range(self.depth     )
							]

							if self.use_linear_projection:
												UpperCAmelCase_					:   Any   =		nn.Dense(__A       ,	dtype=self.dtype     )
							else:
												UpperCAmelCase_					:   Optional[int]   =		nn.Conv(
												    __A       ,	kernel_size=(1, 1)       ,	strides=(1, 1)       ,	padding='VALID'       ,	dtype=self.dtype       ,	)

							UpperCAmelCase_					:   Dict   =		nn.Dropout(rate=self.dropout     )



		def __call__(							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase=True     )			->							Optional[Any]:
							UpperCAmelCase_					:   Optional[Any]   =		hidden_states.shape
							UpperCAmelCase_					:   Dict   =		hidden_states
							UpperCAmelCase_					:   int   =		self.norm(__A     )
							if self.use_linear_projection:
												UpperCAmelCase_					:   Tuple   =		hidden_states.reshape(__A       ,	height * width       ,	__A     )
												UpperCAmelCase_					:   int   =		self.proj_in(__A     )
							else:
												UpperCAmelCase_					:   Dict   =		self.proj_in(__A     )
												UpperCAmelCase_					:   Optional[int]   =		hidden_states.reshape(__A       ,	height * width       ,	__A     )

							for transformer_block in self.transformer_blocks:
												UpperCAmelCase_					:   Dict   =		transformer_block(__A       ,	__A       ,	deterministic=__A     )

							if self.use_linear_projection:
												UpperCAmelCase_					:   List[Any]   =		self.proj_out(__A     )
												UpperCAmelCase_					:   Any   =		hidden_states.reshape(__A       ,	__A       ,	__A       ,	__A     )
							else:
												UpperCAmelCase_					:   Dict   =		hidden_states.reshape(__A       ,	__A       ,	__A       ,	__A     )
												UpperCAmelCase_					:   List[str]   =		self.proj_out(__A     )

							UpperCAmelCase_					:   Any   =		hidden_states + residual
							return self.dropout_layer(__A       ,	deterministic=__A     )





class 						lowerCamelCase   (nn.Module    ):







		'''simple docstring'''



		_snake_case					:					Optional[int] 		=						4_2
		_snake_case					:					List[Any] 		=						0.0
		_snake_case					:					int 		=						jnp.floataa



		def       __UpperCAmelCase    (							self     )			->							int:
							UpperCAmelCase_					:   Union[str, Any]   =		FlaxGEGLU(self.dim       ,	self.dropout       ,	self.dtype     )
							UpperCAmelCase_					:   Any   =		nn.Dense(self.dim       ,	dtype=self.dtype     )



		def __call__(							self       ,	_UpperCamelCase       ,	_UpperCamelCase=True     )			->							Tuple:
							UpperCAmelCase_					:   str   =		self.net_a(__A       ,	deterministic=__A     )
							UpperCAmelCase_					:   Union[str, Any]   =		self.net_a(__A     )
							return hidden_states





class 						lowerCamelCase   (nn.Module    ):







		'''simple docstring'''



		_snake_case					:					Optional[Any] 		=						4_2
		_snake_case					:					Any 		=						0.0
		_snake_case					:					Optional[int] 		=						jnp.floataa



		def       __UpperCAmelCase    (							self     )			->							List[Any]:
							UpperCAmelCase_					:   List[str]   =		self.dim * 4
							UpperCAmelCase_					:   Dict   =		nn.Dense(inner_dim * 2       ,	dtype=self.dtype     )
							UpperCAmelCase_					:   List[str]   =		nn.Dropout(rate=self.dropout     )



		def __call__(							self       ,	_UpperCamelCase       ,	_UpperCamelCase=True     )			->							Optional[int]:
							UpperCAmelCase_					:   Any   =		self.proj(__A     )
							UpperCAmelCase_					:   List[str]   =		jnp.split(__A       ,	2       ,	axis=2     )
							return self.dropout_layer(hidden_linear * nn.gelu(__A     )       ,	deterministic=__A     )














from __future__ import annotations



class 						lowerCamelCase   :







		'''simple docstring'''



		def __init__(							self       ,	_UpperCamelCase = 0     )			->							None:
							UpperCAmelCase_					:   int   =		value
							UpperCAmelCase_					:   Node | None   =		None
							UpperCAmelCase_					:   Node | None   =		None





def   lowercase__   (  __snake_case					:							Node | None  , __snake_case					:							Node | None						):


					'''simple docstring'''


					if treea is None:
										return treea
					if treea is None:
										return treea

					UpperCAmelCase_					:   List[Any]   =		treea.value + treea.value
					UpperCAmelCase_					:   Dict   =		merge_two_binary_trees(treea.left  , treea.left						)
					UpperCAmelCase_					:   str   =		merge_two_binary_trees(treea.right  , treea.right						)
					return treea




def   lowercase__   (  __snake_case					:							Node | None						):


					'''simple docstring'''


					if root:
										print(root.value						)
										print_preorder(root.left						)
										print_preorder(root.right						)


if __name__ == "__main__":
		__UpperCAmelCase										=		Node(1)
		__UpperCAmelCase										=		Node(2)
		__UpperCAmelCase										=		Node(3)
		__UpperCAmelCase										=		Node(4)

		__UpperCAmelCase										=		Node(2)
		__UpperCAmelCase										=		Node(4)
		__UpperCAmelCase										=		Node(6)
		__UpperCAmelCase										=		Node(9)
		__UpperCAmelCase										=		Node(5)

		print('Tree1 is: ')
		print_preorder(treea)
		print('Tree2 is: ')
		print_preorder(treea)
		__UpperCAmelCase										=		merge_two_binary_trees(treea, treea)
		print('Merged Tree is: ')
		print_preorder(merged_tree)






