import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp

def 	__lowerCamelCase							(				__a    : List[Any]      ,							__a    : List[Any]      ,							__a    : int      ,							__a    : Optional[int]      ,							__a    : int = 4_096		)      ->       Optional[Any]:
	_lowercase   ,  _lowercase   ,  _lowercase   =key.shape[-3:]
	_lowercase   =value.shape[-1]
	_lowercase   =min(__a      ,							__a		)
	_lowercase   =query / jnp.sqrt(__a		)

	@functools.partial(jax.checkpoint      ,							prevent_cse=__a		)
	def summarize_chunk(__a    : Dict      ,							__a    : Optional[Any]      ,							__a    : Optional[int]		):
		_lowercase   =jnp.einsum("...qhd,...khd->...qhk"      ,							__a      ,							__a      ,							precision=__a		)

		_lowercase   =jnp.max(__a      ,							axis=-1      ,							keepdims=__a		)
		_lowercase   =jax.lax.stop_gradient(__a		)
		_lowercase   =jnp.exp(attn_weights - max_score		)

		_lowercase   =jnp.einsum("...vhf,...qhv->...qhf"      ,							__a      ,							__a      ,							precision=__a		)
		_lowercase   =jnp.einsum("...qhk->...qh"      ,							__a		)

		return (exp_values, exp_weights.sum(axis=-1		), max_score)

	def chunk_scanner(__a    : str		):
		# julienne key array
		_lowercase   =jax.lax.dynamic_slice(
		    operand=__a      ,							start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]      ,							slice_sizes=list(key.shape[:-3]		) + [key_chunk_size, num_heads, k_features]      ,							)

		# julienne value array
		_lowercase   =jax.lax.dynamic_slice(
		    operand=__a      ,							start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]      ,							slice_sizes=list(value.shape[:-3]		) + [key_chunk_size, num_heads, v_features]      ,							)

		return summarize_chunk(__a      ,							__a      ,							__a		)

	_lowercase   ,  _lowercase   ,  _lowercase   =jax.lax.map(f=__a      ,							xs=jnp.arange(0      ,							__a      ,							__a		)		)

	_lowercase   =jnp.max(__a      ,							axis=0      ,							keepdims=__a		)
	_lowercase   =jnp.exp(chunk_max - global_max		)

	chunk_values *= jnp.expand_dims(__a      ,							axis=-1		)
	chunk_weights *= max_diffs

	_lowercase   =chunk_values.sum(axis=0		)
	_lowercase   =jnp.expand_dims(__a      ,							-1		).sum(axis=0		)

	return all_values / all_weights

def 	__lowerCamelCase							(				__a    : Union[str, Any]      ,							__a    : Dict      ,							__a    : int      ,							__a    : Any=jax.lax.Precision.HIGHEST      ,							__a    : int = 1_024      ,							__a    : int = 4_096		)      ->       Any:
	_lowercase   ,  _lowercase   ,  _lowercase   =query.shape[-3:]

	def chunk_scanner(__a    : List[Any]      ,							__a    : Union[str, Any]		):
		# julienne query array
		_lowercase   =jax.lax.dynamic_slice(
		    operand=__a      ,							start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]      ,							slice_sizes=list(query.shape[:-3]		) + [min(__a      ,							__a		), num_heads, q_features]      ,							)

		return (
		    chunk_idx + query_chunk_size,  # unused ignore it
		    _query_chunk_attention(
		        query=__a      ,							key=__a      ,							value=__a      ,							precision=__a      ,							key_chunk_size=__a		),
		)

	_lowercase   ,  _lowercase   =jax.lax.scan(
	    f=__a      ,							init=0      ,							xs=__a      ,							length=math.ceil(num_q / query_chunk_size		)		)

	return jnp.concatenate(__a      ,							axis=-3		)  # fuse the chunked result back
class 	_a  (							nn.Module					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					8
	__SCREAMING_SNAKE_CASE              =					64
	__SCREAMING_SNAKE_CASE              =					0.0
	__SCREAMING_SNAKE_CASE              =					False
	__SCREAMING_SNAKE_CASE              =					jnp.floataa


	def __lowerCAmelCase      (   self       ):
		_lowercase   =self.dim_head * self.heads
		_lowercase   =self.dim_head**-0.5

		# Weights were exported with old names {to_q, to_k, to_v, to_out}
		_lowercase   =nn.Dense(lowerCAmelCase_       ,  use_bias=lowerCAmelCase_       ,  dtype=self.dtype       ,  name="to_q"       )
		_lowercase   =nn.Dense(lowerCAmelCase_       ,  use_bias=lowerCAmelCase_       ,  dtype=self.dtype       ,  name="to_k"       )
		_lowercase   =nn.Dense(lowerCAmelCase_       ,  use_bias=lowerCAmelCase_       ,  dtype=self.dtype       ,  name="to_v"       )

		_lowercase   =nn.Dense(self.query_dim       ,  dtype=self.dtype       ,  name="to_out_0"       )
		_lowercase   =nn.Dropout(rate=self.dropout       )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   ,  _lowercase   ,  _lowercase   =tensor.shape
		_lowercase   =self.heads
		_lowercase   =tensor.reshape(lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  dim // head_size       )
		_lowercase   =jnp.transpose(lowerCAmelCase_       ,  (0, 2, 1, 3)       )
		_lowercase   =tensor.reshape(batch_size * head_size       ,  lowerCAmelCase_       ,  dim // head_size       )
		return tensor


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   ,  _lowercase   ,  _lowercase   =tensor.shape
		_lowercase   =self.heads
		_lowercase   =tensor.reshape(batch_size // head_size       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       )
		_lowercase   =jnp.transpose(lowerCAmelCase_       ,  (0, 2, 1, 3)       )
		_lowercase   =tensor.reshape(batch_size // head_size       ,  lowerCAmelCase_       ,  dim * head_size       )
		return tensor


	def __call__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_=True       ):
		_lowercase   =hidden_states if context is None else context

		_lowercase   =self.query(lowerCAmelCase_       )
		_lowercase   =self.key(lowerCAmelCase_       )
		_lowercase   =self.value(lowerCAmelCase_       )

		_lowercase   =self.reshape_heads_to_batch_dim(lowerCAmelCase_       )
		_lowercase   =self.reshape_heads_to_batch_dim(lowerCAmelCase_       )
		_lowercase   =self.reshape_heads_to_batch_dim(lowerCAmelCase_       )

		if self.use_memory_efficient_attention:
			_lowercase   =query_states.transpose(1       ,  0       ,  2       )
			_lowercase   =key_states.transpose(1       ,  0       ,  2       )
			_lowercase   =value_states.transpose(1       ,  0       ,  2       )

			# this if statement create a chunk size for each layer of the unet
			# the chunk size is equal to the query_length dimension of the deepest layer of the unet

			_lowercase   =query_states.shape[-3]
			if flatten_latent_dim % 64 == 0:
				_lowercase   =int(flatten_latent_dim / 64       )
			elif flatten_latent_dim % 16 == 0:
				_lowercase   =int(flatten_latent_dim / 16       )
			elif flatten_latent_dim % 4 == 0:
				_lowercase   =int(flatten_latent_dim / 4       )
			else:
				_lowercase   =int(lowerCAmelCase_       )

			_lowercase   =jax_memory_efficient_attention(
			    lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  query_chunk_size=lowerCAmelCase_       ,  key_chunk_size=4096 * 4       )

			_lowercase   =hidden_states.transpose(1       ,  0       ,  2       )
		else:
			# compute attentions
			_lowercase   =jnp.einsum("b i d, b j d->b i j"       ,  lowerCAmelCase_       ,  lowerCAmelCase_       )
			_lowercase   =attention_scores * self.scale
			_lowercase   =nn.softmax(lowerCAmelCase_       ,  axis=2       )

			# attend to values
			_lowercase   =jnp.einsum("b i j, b j d -> b i d"       ,  lowerCAmelCase_       ,  lowerCAmelCase_       )

		_lowercase   =self.reshape_batch_dim_to_heads(lowerCAmelCase_       )
		_lowercase   =self.proj_attn(lowerCAmelCase_       )
		return self.dropout_layer(lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )

class 	_a  (							nn.Module					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					0.0
	__SCREAMING_SNAKE_CASE              =					False
	__SCREAMING_SNAKE_CASE              =					jnp.floataa
	__SCREAMING_SNAKE_CASE              =					False


	def __lowerCAmelCase      (   self       ):
		# self attention (or cross_attention if only_cross_attention is True)
		_lowercase   =FlaxAttention(
		    self.dim       ,  self.n_heads       ,  self.d_head       ,  self.dropout       ,  self.use_memory_efficient_attention       ,  dtype=self.dtype       )
		# cross attention
		_lowercase   =FlaxAttention(
		    self.dim       ,  self.n_heads       ,  self.d_head       ,  self.dropout       ,  self.use_memory_efficient_attention       ,  dtype=self.dtype       )
		_lowercase   =FlaxFeedForward(dim=self.dim       ,  dropout=self.dropout       ,  dtype=self.dtype       )
		_lowercase   =nn.LayerNorm(epsilon=1e-5       ,  dtype=self.dtype       )
		_lowercase   =nn.LayerNorm(epsilon=1e-5       ,  dtype=self.dtype       )
		_lowercase   =nn.LayerNorm(epsilon=1e-5       ,  dtype=self.dtype       )
		_lowercase   =nn.Dropout(rate=self.dropout       )


	def __call__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_=True       ):
		# self attention
		_lowercase   =hidden_states
		if self.only_cross_attention:
			_lowercase   =self.attna(self.norma(lowerCAmelCase_       )       ,  lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )
		else:
			_lowercase   =self.attna(self.norma(lowerCAmelCase_       )       ,  deterministic=lowerCAmelCase_       )
		_lowercase   =hidden_states + residual

		# cross attention
		_lowercase   =hidden_states
		_lowercase   =self.attna(self.norma(lowerCAmelCase_       )       ,  lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )
		_lowercase   =hidden_states + residual

		# feed forward
		_lowercase   =hidden_states
		_lowercase   =self.ff(self.norma(lowerCAmelCase_       )       ,  deterministic=lowerCAmelCase_       )
		_lowercase   =hidden_states + residual

		return self.dropout_layer(lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )

class 	_a  (							nn.Module					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					1
	__SCREAMING_SNAKE_CASE              =					0.0
	__SCREAMING_SNAKE_CASE              =					False
	__SCREAMING_SNAKE_CASE              =					False
	__SCREAMING_SNAKE_CASE              =					jnp.floataa
	__SCREAMING_SNAKE_CASE              =					False


	def __lowerCAmelCase      (   self       ):
		_lowercase   =nn.GroupNorm(num_groups=32       ,  epsilon=1e-5       )

		_lowercase   =self.n_heads * self.d_head
		if self.use_linear_projection:
			_lowercase   =nn.Dense(lowerCAmelCase_       ,  dtype=self.dtype       )
		else:
			_lowercase   =nn.Conv(
			    lowerCAmelCase_       ,  kernel_size=(1, 1)       ,  strides=(1, 1)       ,  padding="VALID"       ,  dtype=self.dtype       ,  )

		_lowercase   =[
		    FlaxBasicTransformerBlock(
		        lowerCAmelCase_       ,  self.n_heads       ,  self.d_head       ,  dropout=self.dropout       ,  only_cross_attention=self.only_cross_attention       ,  dtype=self.dtype       ,  use_memory_efficient_attention=self.use_memory_efficient_attention       ,  )
		    for _ in range(self.depth       )
		]

		if self.use_linear_projection:
			_lowercase   =nn.Dense(lowerCAmelCase_       ,  dtype=self.dtype       )
		else:
			_lowercase   =nn.Conv(
			    lowerCAmelCase_       ,  kernel_size=(1, 1)       ,  strides=(1, 1)       ,  padding="VALID"       ,  dtype=self.dtype       ,  )

		_lowercase   =nn.Dropout(rate=self.dropout       )


	def __call__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_=True       ):
		_lowercase   ,  _lowercase   ,  _lowercase   ,  _lowercase   =hidden_states.shape
		_lowercase   =hidden_states
		_lowercase   =self.norm(lowerCAmelCase_       )
		if self.use_linear_projection:
			_lowercase   =hidden_states.reshape(lowerCAmelCase_       ,  height * width       ,  lowerCAmelCase_       )
			_lowercase   =self.proj_in(lowerCAmelCase_       )
		else:
			_lowercase   =self.proj_in(lowerCAmelCase_       )
			_lowercase   =hidden_states.reshape(lowerCAmelCase_       ,  height * width       ,  lowerCAmelCase_       )

		for transformer_block in self.transformer_blocks:
			_lowercase   =transformer_block(lowerCAmelCase_       ,  lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )

		if self.use_linear_projection:
			_lowercase   =self.proj_out(lowerCAmelCase_       )
			_lowercase   =hidden_states.reshape(lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       )
		else:
			_lowercase   =hidden_states.reshape(lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       )
			_lowercase   =self.proj_out(lowerCAmelCase_       )

		_lowercase   =hidden_states + residual
		return self.dropout_layer(lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )

class 	_a  (							nn.Module					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					0.0
	__SCREAMING_SNAKE_CASE              =					jnp.floataa


	def __lowerCAmelCase      (   self       ):
		# The second linear layer needs to be called
		# net_2 for now to match the index of the Sequential layer
		_lowercase   =FlaxGEGLU(self.dim       ,  self.dropout       ,  self.dtype       )
		_lowercase   =nn.Dense(self.dim       ,  dtype=self.dtype       )


	def __call__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_=True       ):
		_lowercase   =self.net_a(lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )
		_lowercase   =self.net_a(lowerCAmelCase_       )
		return hidden_states

class 	_a  (							nn.Module					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					0.0
	__SCREAMING_SNAKE_CASE              =					jnp.floataa


	def __lowerCAmelCase      (   self       ):
		_lowercase   =self.dim * 4
		_lowercase   =nn.Dense(inner_dim * 2       ,  dtype=self.dtype       )
		_lowercase   =nn.Dropout(rate=self.dropout       )


	def __call__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_=True       ):
		_lowercase   =self.proj(lowerCAmelCase_       )
		_lowercase   ,  _lowercase   =jnp.split(lowerCAmelCase_       ,  2       ,  axis=2       )
		return self.dropout_layer(hidden_linear * nn.gelu(lowerCAmelCase_       )       ,  deterministic=lowerCAmelCase_       )











import os
import shutil
import unittest
from unittest.mock import patch

from transformers.testing_utils import CaptureStd, is_pt_tf_cross_test, require_torch
class SCREAMING_SNAKE_CASE_    (       unittest.TestCase       ):






						@patch('sys.argv'		, ['fakeprogrampath', 'env']						)
						def UpperCAmelCase_					(   self   :   Tuple						)				->     Dict:







							"""simple docstring"""


							import transformers.commands.transformers_cli

							with CaptureStd() as cs:
								transformers.commands.transformers_cli.main()
							self.assertIn('Python version'		, cs.out						)
							self.assertIn('Platform'		, cs.out						)
							self.assertIn('Using distributed or parallel set-up in script?'		, cs.out						)






						@is_pt_tf_cross_test
						@patch(
						    'sys.argv'		, ['fakeprogrampath', 'pt-to-tf', '--model-name', 'hf-internal-testing/tiny-random-gptj', '--no-pr']						)
						def UpperCAmelCase_					(   self   :   Union[str, Any]						)				->     Optional[Any]:







							"""simple docstring"""


							import transformers.commands.transformers_cli

							shutil.rmtree('/tmp/hf-internal-testing/tiny-random-gptj'		, ignore_errors=_A						)  # cleans potential past runs
							transformers.commands.transformers_cli.main()

							# The original repo has no TF weights -- if they exist, they were created by the CLI
							self.assertTrue(os.path.exists('/tmp/hf-internal-testing/tiny-random-gptj/tf_model.h5'						)						)






						@require_torch
						@patch('sys.argv'		, ['fakeprogrampath', 'download', 'hf-internal-testing/tiny-random-gptj', '--cache-dir', '/tmp']						)
						def UpperCAmelCase_					(   self   :   List[Any]						)				->     str:







							"""simple docstring"""


							import transformers.commands.transformers_cli

							# # remove any previously downloaded model to start clean
							shutil.rmtree('/tmp/models--hf-internal-testing--tiny-random-gptj'		, ignore_errors=_A						)

							# run the command
							transformers.commands.transformers_cli.main()

							# check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--tiny-random-gptj
							self.assertTrue(os.path.exists('/tmp/models--hf-internal-testing--tiny-random-gptj/blobs'						)						)
							self.assertTrue(os.path.exists('/tmp/models--hf-internal-testing--tiny-random-gptj/refs'						)						)
							self.assertTrue(os.path.exists('/tmp/models--hf-internal-testing--tiny-random-gptj/snapshots'						)						)






						@require_torch
						@patch(
						    'sys.argv'		, [
						        'fakeprogrampath',
						        'download',
						        'hf-internal-testing/test_dynamic_model_with_tokenizer',
						        '--trust-remote-code',
						        '--cache-dir',
						        '/tmp',
						    ]		, )
						def UpperCAmelCase_					(   self   :   Optional[Any]						)				->     str:







							"""simple docstring"""


							import transformers.commands.transformers_cli

							# # remove any previously downloaded model to start clean
							shutil.rmtree('/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer'		, ignore_errors=_A						)

							# run the command
							transformers.commands.transformers_cli.main()

							# check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer
							self.assertTrue(os.path.exists('/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/blobs'						)						)
							self.assertTrue(os.path.exists('/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/refs'						)						)
							self.assertTrue(
							    os.path.exists('/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/snapshots'						)						)

