




import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp




def    lowerCamelCase__			(		A__			:   List[Any]    ,     A__			:   Optional[int]    ,     A__			:   List[str]    ,     A__			:   Any    ,     A__			:   Union[str, Any] = 4096      ):





       '''simple docstring'''



       __lowerCamelCase  =		key.shape[-3:]
       __lowerCamelCase  =		value.shape[-1]
       __lowerCamelCase  =		min(__snake_case    ,     __snake_case      )
       __lowerCamelCase  =		query / jnp.sqrt(__snake_case      )

       @functools.partial(jax.checkpoint    ,     prevent_cse=__snake_case      )
       def summarize_chunk(A__			:   Tuple    ,     A__			:   Optional[Any]    ,     A__			:   List[str]      ):
              __lowerCamelCase  =		jnp.einsum("""...qhd,...khd->...qhk"""    ,     __snake_case    ,     __snake_case    ,     precision=__snake_case      )

              __lowerCamelCase  =		jnp.max(__snake_case    ,     axis=-1    ,     keepdims=__snake_case      )
              __lowerCamelCase  =		jax.lax.stop_gradient(__snake_case      )
              __lowerCamelCase  =		jnp.exp(attn_weights - max_score      )

              __lowerCamelCase  =		jnp.einsum("""...vhf,...qhv->...qhf"""    ,     __snake_case    ,     __snake_case    ,     precision=__snake_case      )
              __lowerCamelCase  =		jnp.einsum("""...qhk->...qh"""    ,     __snake_case      )

              return (exp_values, exp_weights.sum(axis=-1      ), max_score)

       def chunk_scanner(A__			:   Any      ):
              # julienne key array
              __lowerCamelCase  =		jax.lax.dynamic_slice(
                  operand=__snake_case    ,     start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]    ,     slice_sizes=list(key.shape[:-3]      ) + [key_chunk_size, num_heads, k_features]    ,     )

              # julienne value array
              __lowerCamelCase  =		jax.lax.dynamic_slice(
                  operand=__snake_case    ,     start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]    ,     slice_sizes=list(value.shape[:-3]      ) + [key_chunk_size, num_heads, v_features]    ,     )

              return summarize_chunk(__snake_case    ,     __snake_case    ,     __snake_case      )

       __lowerCamelCase  =		jax.lax.map(f=__snake_case    ,     xs=jnp.arange(0    ,     __snake_case    ,     __snake_case      )      )

       __lowerCamelCase  =		jnp.max(__snake_case    ,     axis=0    ,     keepdims=__snake_case      )
       __lowerCamelCase  =		jnp.exp(chunk_max - global_max      )

       chunk_values *= jnp.expand_dims(__snake_case    ,     axis=-1      )
       chunk_weights *= max_diffs

       __lowerCamelCase  =		chunk_values.sum(axis=0      )
       __lowerCamelCase  =		jnp.expand_dims(__snake_case    ,     -1      ).sum(axis=0      )

       return all_values / all_weights




def    lowerCamelCase__			(		A__			:   Dict    ,     A__			:   int    ,     A__			:   Union[str, Any]    ,     A__			:   Optional[Any]=jax.lax.Precision.HIGHEST    ,     A__			:   Dict = 1024    ,     A__			:   List[str] = 4096      ):





       '''simple docstring'''



       __lowerCamelCase  =		query.shape[-3:]

       def chunk_scanner(A__			:   Tuple    ,     A__			:   Optional[Any]      ):
              # julienne query array
              __lowerCamelCase  =		jax.lax.dynamic_slice(
                  operand=__snake_case    ,     start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]    ,     slice_sizes=list(query.shape[:-3]      ) + [min(__snake_case    ,     __snake_case      ), num_heads, q_features]    ,     )

              return (
                  chunk_idx + query_chunk_size,  # unused ignore it
                  _query_chunk_attention(
                      query=__snake_case    ,     key=__snake_case    ,     value=__snake_case    ,     precision=__snake_case    ,     key_chunk_size=__snake_case      ),
              )

       __lowerCamelCase  =		jax.lax.scan(
           f=__snake_case    ,     init=0    ,     xs=__snake_case    ,     length=math.ceil(num_q / query_chunk_size      )      )

       return jnp.concatenate(__snake_case    ,     axis=-3      )  # fuse the chunked result back





class        lowerCamelCase__(   nn.Module):
      UpperCAmelCase__ :      List[str]       						=  42
      UpperCAmelCase__ :      Any       						=  8
      UpperCAmelCase__ :      int       						=  64
      UpperCAmelCase__ :      List[str]       						=  0.0
      UpperCAmelCase__ :      str       						=  False
      UpperCAmelCase__ :      Tuple       						=  jnp.floataa
      def      lowerCAmelCase__		(							self:					List[Any]  ):
             __lowerCamelCase  =		self.dim_head * self.heads
             __lowerCamelCase  =		self.dim_head**-0.5

             # Weights were exported with old names {to_q, to_k, to_v, to_out}
             __lowerCamelCase  =		nn.Dense(UpperCamelCase_       ,			use_bias=UpperCamelCase_       ,			dtype=self.dtype       ,			name="""to_q"""  )
             __lowerCamelCase  =		nn.Dense(UpperCamelCase_       ,			use_bias=UpperCamelCase_       ,			dtype=self.dtype       ,			name="""to_k"""  )
             __lowerCamelCase  =		nn.Dense(UpperCamelCase_       ,			use_bias=UpperCamelCase_       ,			dtype=self.dtype       ,			name="""to_v"""  )

             __lowerCamelCase  =		nn.Dense(self.query_dim       ,			dtype=self.dtype       ,			name="""to_out_0"""  )
             __lowerCamelCase  =		nn.Dropout(rate=self.dropout  )
      def      lowerCAmelCase__		(							self:					Union[str, Any]       ,			UpperCamelCase_:					Optional[int]  ):
             __lowerCamelCase  =		tensor.shape
             __lowerCamelCase  =		self.heads
             __lowerCamelCase  =		tensor.reshape(UpperCamelCase_       ,			UpperCamelCase_       ,			UpperCamelCase_       ,			dim // head_size  )
             __lowerCamelCase  =		jnp.transpose(UpperCamelCase_       ,			(0, 2, 1, 3)  )
             __lowerCamelCase  =		tensor.reshape(batch_size * head_size       ,			UpperCamelCase_       ,			dim // head_size  )
             return tensor
      def      lowerCAmelCase__		(							self:					Optional[Any]       ,			UpperCamelCase_:					int  ):
             __lowerCamelCase  =		tensor.shape
             __lowerCamelCase  =		self.heads
             __lowerCamelCase  =		tensor.reshape(batch_size // head_size       ,			UpperCamelCase_       ,			UpperCamelCase_       ,			UpperCamelCase_  )
             __lowerCamelCase  =		jnp.transpose(UpperCamelCase_       ,			(0, 2, 1, 3)  )
             __lowerCamelCase  =		tensor.reshape(batch_size // head_size       ,			UpperCamelCase_       ,			dim * head_size  )
             return tensor




      def __call__(							self:					Dict       ,			UpperCamelCase_:					Any       ,			UpperCamelCase_:					Optional[Any]=None       ,			UpperCamelCase_:					Union[str, Any]=True  ):
             __lowerCamelCase  =		hidden_states if context is None else context

             __lowerCamelCase  =		self.query(UpperCamelCase_  )
             __lowerCamelCase  =		self.key(UpperCamelCase_  )
             __lowerCamelCase  =		self.value(UpperCamelCase_  )

             __lowerCamelCase  =		self.reshape_heads_to_batch_dim(UpperCamelCase_  )
             __lowerCamelCase  =		self.reshape_heads_to_batch_dim(UpperCamelCase_  )
             __lowerCamelCase  =		self.reshape_heads_to_batch_dim(UpperCamelCase_  )

             if self.use_memory_efficient_attention:
                    __lowerCamelCase  =		query_states.transpose(1       ,			0       ,			2  )
                    __lowerCamelCase  =		key_states.transpose(1       ,			0       ,			2  )
                    __lowerCamelCase  =		value_states.transpose(1       ,			0       ,			2  )

                    # this if statement create a chunk size for each layer of the unet
                    # the chunk size is equal to the query_length dimension of the deepest layer of the unet

                    __lowerCamelCase  =		query_states.shape[-3]
                    if flatten_latent_dim % 64 == 0:
                           __lowerCamelCase  =		int(flatten_latent_dim / 64  )
                    elif flatten_latent_dim % 16 == 0:
                           __lowerCamelCase  =		int(flatten_latent_dim / 16  )
                    elif flatten_latent_dim % 4 == 0:
                           __lowerCamelCase  =		int(flatten_latent_dim / 4  )
                    else:
                           __lowerCamelCase  =		int(UpperCamelCase_  )

                    __lowerCamelCase  =		jax_memory_efficient_attention(
                        UpperCamelCase_       ,			UpperCamelCase_       ,			UpperCamelCase_       ,			query_chunk_size=UpperCamelCase_       ,			key_chunk_size=40_96 * 4  )

                    __lowerCamelCase  =		hidden_states.transpose(1       ,			0       ,			2  )
             else:
                    # compute attentions
                    __lowerCamelCase  =		jnp.einsum("""b i d, b j d->b i j"""       ,			UpperCamelCase_       ,			UpperCamelCase_  )
                    __lowerCamelCase  =		attention_scores * self.scale
                    __lowerCamelCase  =		nn.softmax(UpperCamelCase_       ,			axis=2  )

                    # attend to values
                    __lowerCamelCase  =		jnp.einsum("""b i j, b j d -> b i d"""       ,			UpperCamelCase_       ,			UpperCamelCase_  )

             __lowerCamelCase  =		self.reshape_batch_dim_to_heads(UpperCamelCase_  )
             __lowerCamelCase  =		self.proj_attn(UpperCamelCase_  )
             return self.dropout_layer(UpperCamelCase_       ,			deterministic=UpperCamelCase_  )





class        lowerCamelCase__(   nn.Module):
      UpperCAmelCase__ :      List[Any]       						=  42
      UpperCAmelCase__ :      List[str]       						=  42
      UpperCAmelCase__ :      List[Any]       						=  42
      UpperCAmelCase__ :      Optional[int]       						=  0.0
      UpperCAmelCase__ :      Optional[int]       						=  False
      UpperCAmelCase__ :      Any       						=  jnp.floataa
      UpperCAmelCase__ :      Optional[Any]       						=  False
      def      lowerCAmelCase__		(							self:					List[str]  ):
             __lowerCamelCase  =		FlaxAttention(
                 self.dim       ,			self.n_heads       ,			self.d_head       ,			self.dropout       ,			self.use_memory_efficient_attention       ,			dtype=self.dtype  )
             # cross attention
             __lowerCamelCase  =		FlaxAttention(
                 self.dim       ,			self.n_heads       ,			self.d_head       ,			self.dropout       ,			self.use_memory_efficient_attention       ,			dtype=self.dtype  )
             __lowerCamelCase  =		FlaxFeedForward(dim=self.dim       ,			dropout=self.dropout       ,			dtype=self.dtype  )
             __lowerCamelCase  =		nn.LayerNorm(epsilon=1E-5       ,			dtype=self.dtype  )
             __lowerCamelCase  =		nn.LayerNorm(epsilon=1E-5       ,			dtype=self.dtype  )
             __lowerCamelCase  =		nn.LayerNorm(epsilon=1E-5       ,			dtype=self.dtype  )
             __lowerCamelCase  =		nn.Dropout(rate=self.dropout  )




      def __call__(							self:					Union[str, Any]       ,			UpperCamelCase_:					int       ,			UpperCamelCase_:					Dict       ,			UpperCamelCase_:					Dict=True  ):
             __lowerCamelCase  =		hidden_states
             if self.only_cross_attention:
                    __lowerCamelCase  =		self.attna(self.norma(UpperCamelCase_  )       ,			UpperCamelCase_       ,			deterministic=UpperCamelCase_  )
             else:
                    __lowerCamelCase  =		self.attna(self.norma(UpperCamelCase_  )       ,			deterministic=UpperCamelCase_  )
             __lowerCamelCase  =		hidden_states + residual

             # cross attention
             __lowerCamelCase  =		hidden_states
             __lowerCamelCase  =		self.attna(self.norma(UpperCamelCase_  )       ,			UpperCamelCase_       ,			deterministic=UpperCamelCase_  )
             __lowerCamelCase  =		hidden_states + residual

             # feed forward
             __lowerCamelCase  =		hidden_states
             __lowerCamelCase  =		self.ff(self.norma(UpperCamelCase_  )       ,			deterministic=UpperCamelCase_  )
             __lowerCamelCase  =		hidden_states + residual

             return self.dropout_layer(UpperCamelCase_       ,			deterministic=UpperCamelCase_  )





class        lowerCamelCase__(   nn.Module):
      UpperCAmelCase__ :      Optional[int]       						=  42
      UpperCAmelCase__ :      Tuple       						=  42
      UpperCAmelCase__ :      Union[str, Any]       						=  42
      UpperCAmelCase__ :      List[str]       						=  1
      UpperCAmelCase__ :      List[str]       						=  0.0
      UpperCAmelCase__ :      Optional[int]       						=  False
      UpperCAmelCase__ :      Union[str, Any]       						=  False
      UpperCAmelCase__ :      Optional[int]       						=  jnp.floataa
      UpperCAmelCase__ :      Tuple       						=  False
      def      lowerCAmelCase__		(							self:					str  ):
             __lowerCamelCase  =		nn.GroupNorm(num_groups=32       ,			epsilon=1E-5  )

             __lowerCamelCase  =		self.n_heads * self.d_head
             if self.use_linear_projection:
                    __lowerCamelCase  =		nn.Dense(UpperCamelCase_       ,			dtype=self.dtype  )
             else:
                    __lowerCamelCase  =		nn.Conv(
                        UpperCamelCase_       ,			kernel_size=(1, 1)       ,			strides=(1, 1)       ,			padding="""VALID"""       ,			dtype=self.dtype       ,			)

             __lowerCamelCase  =		[
                 FlaxBasicTransformerBlock(
                     UpperCamelCase_       ,			self.n_heads       ,			self.d_head       ,			dropout=self.dropout       ,			only_cross_attention=self.only_cross_attention       ,			dtype=self.dtype       ,			use_memory_efficient_attention=self.use_memory_efficient_attention       ,			)
                 for _ in range(self.depth  )
             ]

             if self.use_linear_projection:
                    __lowerCamelCase  =		nn.Dense(UpperCamelCase_       ,			dtype=self.dtype  )
             else:
                    __lowerCamelCase  =		nn.Conv(
                        UpperCamelCase_       ,			kernel_size=(1, 1)       ,			strides=(1, 1)       ,			padding="""VALID"""       ,			dtype=self.dtype       ,			)

             __lowerCamelCase  =		nn.Dropout(rate=self.dropout  )




      def __call__(							self:					Dict       ,			UpperCamelCase_:					Union[str, Any]       ,			UpperCamelCase_:					Any       ,			UpperCamelCase_:					List[str]=True  ):
             __lowerCamelCase  =		hidden_states.shape
             __lowerCamelCase  =		hidden_states
             __lowerCamelCase  =		self.norm(UpperCamelCase_  )
             if self.use_linear_projection:
                    __lowerCamelCase  =		hidden_states.reshape(UpperCamelCase_       ,			height * width       ,			UpperCamelCase_  )
                    __lowerCamelCase  =		self.proj_in(UpperCamelCase_  )
             else:
                    __lowerCamelCase  =		self.proj_in(UpperCamelCase_  )
                    __lowerCamelCase  =		hidden_states.reshape(UpperCamelCase_       ,			height * width       ,			UpperCamelCase_  )

             for transformer_block in self.transformer_blocks:
                    __lowerCamelCase  =		transformer_block(UpperCamelCase_       ,			UpperCamelCase_       ,			deterministic=UpperCamelCase_  )

             if self.use_linear_projection:
                    __lowerCamelCase  =		self.proj_out(UpperCamelCase_  )
                    __lowerCamelCase  =		hidden_states.reshape(UpperCamelCase_       ,			UpperCamelCase_       ,			UpperCamelCase_       ,			UpperCamelCase_  )
             else:
                    __lowerCamelCase  =		hidden_states.reshape(UpperCamelCase_       ,			UpperCamelCase_       ,			UpperCamelCase_       ,			UpperCamelCase_  )
                    __lowerCamelCase  =		self.proj_out(UpperCamelCase_  )

             __lowerCamelCase  =		hidden_states + residual
             return self.dropout_layer(UpperCamelCase_       ,			deterministic=UpperCamelCase_  )





class        lowerCamelCase__(   nn.Module):
      UpperCAmelCase__ :      str       						=  42
      UpperCAmelCase__ :      str       						=  0.0
      UpperCAmelCase__ :      int       						=  jnp.floataa
      def      lowerCAmelCase__		(							self:					Optional[int]  ):
             __lowerCamelCase  =		FlaxGEGLU(self.dim       ,			self.dropout       ,			self.dtype  )
             __lowerCamelCase  =		nn.Dense(self.dim       ,			dtype=self.dtype  )




      def __call__(							self:					Any       ,			UpperCamelCase_:					Any       ,			UpperCamelCase_:					Any=True  ):
             __lowerCamelCase  =		self.net_a(UpperCamelCase_       ,			deterministic=UpperCamelCase_  )
             __lowerCamelCase  =		self.net_a(UpperCamelCase_  )
             return hidden_states





class        lowerCamelCase__(   nn.Module):
      UpperCAmelCase__ :      Any       						=  42
      UpperCAmelCase__ :      List[Any]       						=  0.0
      UpperCAmelCase__ :      List[Any]       						=  jnp.floataa
      def      lowerCAmelCase__		(							self:					int  ):
             __lowerCamelCase  =		self.dim * 4
             __lowerCamelCase  =		nn.Dense(inner_dim * 2       ,			dtype=self.dtype  )
             __lowerCamelCase  =		nn.Dropout(rate=self.dropout  )




      def __call__(							self:					List[Any]       ,			UpperCamelCase_:					List[str]       ,			UpperCamelCase_:					List[Any]=True  ):
             __lowerCamelCase  =		self.proj(UpperCamelCase_  )
             __lowerCamelCase  =		jnp.split(UpperCamelCase_       ,			2       ,			axis=2  )
             return self.dropout_layer(hidden_linear * nn.gelu(UpperCamelCase_  )       ,			deterministic=UpperCamelCase_  )






import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


__lowerCamelCase								=      logging.getLogger(__name__)


@dataclass
class 	snake_case_			:




       """simple docstring"""






       _lowerCamelCase           =					field(
           metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}	)
       _lowerCamelCase           =					field(
           default=lowercase__			,			metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}	)
       _lowerCamelCase           =					field(
           default=lowercase__			,			metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}	)
       _lowerCamelCase           =					field(default=lowercase__			,			metadata={"""help""": """Set this flag to use fast tokenization."""}	)
       # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
       # or just modify its tokenizer_config.json.
       _lowerCamelCase           =					field(
           default=lowercase__			,			metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}			,			)







def  _snake_case			(							)  ->  Union[str, Any]:






      '''simple docstring'''







      UpperCAmelCase_	:						Union[str, Any]					   =		HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )

      if len(sys.argv     ) == 2 and sys.argv[1].endswith(".json"     ):
            # If we pass only one argument to the script and it's the path to a json file,
            # let's parse it to get our arguments.
            UpperCAmelCase_   ,   UpperCAmelCase_   ,   UpperCAmelCase_	:						Any					   =		parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]     )     )
      else:
            UpperCAmelCase_   ,   UpperCAmelCase_   ,   UpperCAmelCase_	:						Union[str, Any]					   =		parser.parse_args_into_dataclasses()

      if (
          os.path.exists(training_args.output_dir     )
          and os.listdir(training_args.output_dir     )
          and training_args.do_train
          and not training_args.overwrite_output_dir
      ):
            raise ValueError(
                F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
                " --overwrite_output_dir to overcome."     )

      # Setup logging
      logging.basicConfig(
          format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"       ,	datefmt="%m/%d/%Y %H:%M:%S"       ,	level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN       ,	)
      logger.warning(
          "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"       ,	training_args.local_rank       ,	training_args.device       ,	training_args.n_gpu       ,	bool(training_args.local_rank != -1     )       ,	training_args.fpaa       ,	)
      # Set the verbosity to info of the Transformers logger (on main process only):
      if is_main_process(training_args.local_rank     ):
            transformers.utils.logging.set_verbosity_info()
            transformers.utils.logging.enable_default_handler()
            transformers.utils.logging.enable_explicit_format()
      logger.info("Training/evaluation parameters %s"       ,	__snake_case     )

      # Prepare Question-Answering task
      # Load pretrained model and tokenizer
      #
      # Distributed training:
      # The .from_pretrained methods guarantee that only one local process can concurrently
      # download model & vocab.

      UpperCAmelCase_	:						Any					   =		AutoConfig.from_pretrained(
          model_args.config_name if model_args.config_name else model_args.model_name_or_path       ,	cache_dir=model_args.cache_dir       ,	)
      UpperCAmelCase_	:						List[str]					   =		AutoTokenizer.from_pretrained(
          model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path       ,	cache_dir=model_args.cache_dir       ,	use_fast=__snake_case       ,	)
      UpperCAmelCase_	:						Optional[int]					   =		AutoModelForQuestionAnswering.from_pretrained(
          model_args.model_name_or_path       ,	from_tf=bool(".ckpt" in model_args.model_name_or_path     )       ,	config=__snake_case       ,	cache_dir=model_args.cache_dir       ,	)

      # Get datasets
      UpperCAmelCase_	:						str					   =		hasattr(model.config       ,	"lang2id"     )
      UpperCAmelCase_	:						Optional[Any]					   =		(
          SquadDataset(
              __snake_case       ,	tokenizer=__snake_case       ,	is_language_sensitive=__snake_case       ,	cache_dir=model_args.cache_dir     )
          if training_args.do_train
          else None
      )
      UpperCAmelCase_	:						Optional[Any]					   =		(
          SquadDataset(
              __snake_case       ,	tokenizer=__snake_case       ,	mode="dev"       ,	is_language_sensitive=__snake_case       ,	cache_dir=model_args.cache_dir       ,	)
          if training_args.do_eval
          else None
      )

      # Data collator
      UpperCAmelCase_	:						Optional[Any]					   =		DataCollatorWithPadding(__snake_case       ,	pad_to_multiple_of=8     ) if training_args.fpaa else None

      # Initialize our Trainer
      UpperCAmelCase_	:						Any					   =		Trainer(
          model=__snake_case       ,	args=__snake_case       ,	train_dataset=__snake_case       ,	eval_dataset=__snake_case       ,	data_collator=__snake_case       ,	)

      # Training
      if training_args.do_train:
            trainer.train(
                model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path     ) else None     )
            trainer.save_model()
            # For convenience, we also re-save the tokenizer to the same directory,
            # so that you can share your model easily on huggingface.co/models =)
            if trainer.is_world_master():
                  tokenizer.save_pretrained(training_args.output_dir     )







def  _snake_case			(							__snake_case     )  ->  List[str]:






      '''simple docstring'''







      main()


if __name__ == "__main__":
   main()
