
"""simple docstring"""
from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


lowercase		:    Union[str, Any]			=       logging.get_logger(__name__)




class  _snake_case	(    nn.Module							):
							SCREAMING_SNAKE_CASE		  =    42
							SCREAMING_SNAKE_CASE		  =    jnp.floataa






							def 				__lowercase	(       self    :      Any     ):






												"""simple docstring"""



												__a:    List[str]		     =       self.config.text_config
												__a:    List[str]		     =       self.config.vision_config

												__a:    Tuple		     =       self.config.projection_dim
												__a:    Optional[int]		     =       text_config.hidden_size
												__a:    Tuple		     =       vision_config.hidden_size

												__a:    Tuple		     =       FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
												__a:    Optional[int]		     =       FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__    ,			__UpperCamelCase     ).module_class

												__a:    List[str]		     =       text_module(__UpperCamelCase    ,			dtype=self.dtype     )
												__a:    Optional[Any]		     =       vision_module(__UpperCamelCase    ,			dtype=self.dtype     )

												__a:    Optional[Any]		     =       nn.Dense(
												    self.projection_dim    ,			dtype=self.dtype    ,			kernel_init=jax.nn.initializers.normal(0.02     )    ,			use_bias=__UpperCamelCase    ,			)
												__a:    List[Any]		     =       nn.Dense(
												    self.projection_dim    ,			dtype=self.dtype    ,			kernel_init=jax.nn.initializers.normal(0.02     )    ,			use_bias=__UpperCamelCase    ,			)
												__a:    str		     =       self.param("""logit_scale"""    ,			jax.nn.initializers.ones    ,			[]     )






							def __call__(       self    :      List[str]    ,			snake_case__    :      Union[str, Any]=None    ,			snake_case__    :      int=None    ,			snake_case__    :      Union[str, Any]=None    ,			snake_case__    :      Union[str, Any]=None    ,			snake_case__    :      List[Any]=None    ,			snake_case__    :      bool = True    ,			snake_case__    :      Any=None    ,			snake_case__    :      int=None    ,			snake_case__    :      Dict=None    ,			):






												"""simple docstring"""



												__a:    Optional[int]		     =       return_dict if return_dict is not None else self.config.return_dict

												__a:    List[Any]		     =       self.vision_model(
												    pixel_values=__UpperCamelCase    ,			deterministic=__UpperCamelCase    ,			output_attentions=__UpperCamelCase    ,			output_hidden_states=__UpperCamelCase    ,			return_dict=__UpperCamelCase    ,			)

												__a:    Dict		     =       self.text_model(
												    input_ids=__UpperCamelCase    ,			attention_mask=__UpperCamelCase    ,			token_type_ids=__UpperCamelCase    ,			position_ids=__UpperCamelCase    ,			deterministic=__UpperCamelCase    ,			output_attentions=__UpperCamelCase    ,			output_hidden_states=__UpperCamelCase    ,			return_dict=__UpperCamelCase    ,			)

												__a:    List[Any]		     =       vision_outputs[1]
												__a:    List[Any]		     =       self.visual_projection(__UpperCamelCase     )

												__a:    str		     =       text_outputs[1]
												__a:    Union[str, Any]		     =       self.text_projection(__UpperCamelCase     )

												# normalized features
												__a:    int		     =       image_embeds / jnp.linalg.norm(__UpperCamelCase    ,			axis=-1    ,			keepdims=__UpperCamelCase     )
												__a:    Optional[int]		     =       text_embeds / jnp.linalg.norm(__UpperCamelCase    ,			axis=-1    ,			keepdims=__UpperCamelCase     )

												# cosine similarity as logits
												__a:    int		     =       jnp.exp(self.logit_scale     )
												__a:    str		     =       jnp.matmul(__UpperCamelCase    ,			image_embeds.T     ) * logit_scale
												__a:    Union[str, Any]		     =       logits_per_text.T

												if not return_dict:
																	return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

												return FlaxCLIPOutput(
												    logits_per_image=__UpperCamelCase    ,			logits_per_text=__UpperCamelCase    ,			text_embeds=__UpperCamelCase    ,			image_embeds=__UpperCamelCase    ,			text_model_output=__UpperCamelCase    ,			vision_model_output=__UpperCamelCase    ,			)






class  _snake_case	(    lowercase__							):
							SCREAMING_SNAKE_CASE		  =    HybridCLIPConfig
							SCREAMING_SNAKE_CASE		  =    FlaxHybridCLIPModule






							def __init__(       self    :      List[str]    ,			snake_case__    :      HybridCLIPConfig    ,			snake_case__    :      Optional[Tuple] = None    ,			snake_case__    :      int = 0    ,			snake_case__    :      jnp.dtype = jnp.floataa    ,			**snake_case__    :      Optional[Any]    ,			):






												"""simple docstring"""



												if input_shape is None:
																	__a:    Any		     =       ((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

												__a:    int		     =       self.module_class(config=__UpperCamelCase    ,			dtype=__UpperCamelCase    ,			**__UpperCamelCase     )
												super().__init__(__UpperCamelCase    ,			__UpperCamelCase    ,			input_shape=__UpperCamelCase    ,			seed=__UpperCamelCase    ,			dtype=__UpperCamelCase     )






							def 				__lowercase	(       self    :      int    ,			snake_case__    :      jax.random.PRNGKey    ,			snake_case__    :      Tuple    ,			snake_case__    :      FrozenDict = None     ):






												"""simple docstring"""



												__a:    Dict		     =       jnp.zeros(input_shape[0]    ,			dtype="""i4"""     )
												__a:    Any		     =       jnp.broadcast_to(jnp.arange(jnp.atleast_ad(__UpperCamelCase     ).shape[-1]     )    ,			input_shape[0]     )
												__a:    str		     =       jnp.ones_like(__UpperCamelCase     )
												__a:    List[str]		     =       jnp.ones_like(__UpperCamelCase     )

												__a:    List[Any]		     =       jax.random.normal(__UpperCamelCase    ,			input_shape[1]     )

												__a:    Dict		     =       jax.random.split(__UpperCamelCase     )
												__a:    Optional[Any]		     =       {'''params''': params_rng, '''dropout''': dropout_rng}

												return self.module.init(__UpperCamelCase    ,			__UpperCamelCase    ,			__UpperCamelCase    ,			__UpperCamelCase    ,			__UpperCamelCase    ,			__UpperCamelCase     )["params"]






							def __call__(       self    :      Optional[Any]    ,			snake_case__    :      Union[str, Any]    ,			snake_case__    :      Optional[int]    ,			snake_case__    :      List[str]=None    ,			snake_case__    :      Optional[Any]=None    ,			snake_case__    :      Union[str, Any]=None    ,			snake_case__    :      dict = None    ,			snake_case__    :      jax.random.PRNGKey = None    ,			snake_case__    :      bool = False    ,			snake_case__    :      Optional[bool] = None    ,			snake_case__    :      Optional[bool] = None    ,			snake_case__    :      Optional[bool] = None    ,			):






												"""simple docstring"""



												__a:    str		     =       output_attentions if output_attentions is not None else self.config.output_attentions
												__a:    List[str]		     =       (
												    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
												)
												__a:    Dict		     =       return_dict if return_dict is not None else self.config.return_dict

												if position_ids is None:
																	__a:    Optional[int]		     =       jnp.broadcast_to(jnp.arange(jnp.atleast_ad(__UpperCamelCase     ).shape[-1]     )    ,			input_ids.shape     )

												if token_type_ids is None:
																	__a:    List[Any]		     =       jnp.zeros_like(__UpperCamelCase     )

												if attention_mask is None:
																	__a:    Tuple		     =       jnp.ones_like(__UpperCamelCase     )

												# Handle any PRNG if needed
												__a:    Tuple		     =       {}
												if dropout_rng is not None:
																	__a:    Optional[int]		     =       dropout_rng

												return self.module.apply(
												    {"""params""": params or self.params}    ,			jnp.array(__UpperCamelCase    ,			dtype="""i4"""     )    ,			jnp.array(__UpperCamelCase    ,			dtype=jnp.floataa     )    ,			jnp.array(__UpperCamelCase    ,			dtype="""i4"""     )    ,			jnp.array(__UpperCamelCase    ,			dtype="""i4"""     )    ,			jnp.array(__UpperCamelCase    ,			dtype="""i4"""     )    ,			not train    ,			__UpperCamelCase    ,			__UpperCamelCase    ,			__UpperCamelCase    ,			rngs=__UpperCamelCase    ,			)






							def 				__lowercase	(       self    :      List[Any]    ,			snake_case__    :      Optional[int]    ,			snake_case__    :      List[Any]=None    ,			snake_case__    :      int=None    ,			snake_case__    :      List[str]=None    ,			snake_case__    :      dict = None    ,			snake_case__    :      jax.random.PRNGKey = None    ,			snake_case__    :      str=False    ,			):






												"""simple docstring"""



												if position_ids is None:
																	__a:    Optional[Any]		     =       jnp.broadcast_to(jnp.arange(jnp.atleast_ad(__UpperCamelCase     ).shape[-1]     )    ,			input_ids.shape     )

												if token_type_ids is None:
																	__a:    Optional[Any]		     =       jnp.zeros_like(__UpperCamelCase     )

												if attention_mask is None:
																	__a:    Union[str, Any]		     =       jnp.ones_like(__UpperCamelCase     )

												# Handle any PRNG if needed
												__a:    Any		     =       {}
												if dropout_rng is not None:
																	__a:    List[Any]		     =       dropout_rng

												def _get_features(snake_case__    :      str    ,			snake_case__    :      List[str]    ,			snake_case__    :      Dict    ,			snake_case__    :      Union[str, Any]    ,			snake_case__    :      Union[str, Any]    ,			snake_case__    :      Optional[Any]     ):
																	__a:    Tuple		     =       module.text_model(
																	    input_ids=__UpperCamelCase    ,			attention_mask=__UpperCamelCase    ,			position_ids=__UpperCamelCase    ,			token_type_ids=__UpperCamelCase    ,			deterministic=__UpperCamelCase    ,			)
																	__a:    str		     =       text_outputs[1]
																	__a:    str		     =       module.text_projection(__UpperCamelCase     )
																	return text_features

												return self.module.apply(
												    {"""params""": params or self.params}    ,			jnp.array(__UpperCamelCase    ,			dtype="""i4"""     )    ,			jnp.array(__UpperCamelCase    ,			dtype="""i4"""     )    ,			jnp.array(__UpperCamelCase    ,			dtype="""i4"""     )    ,			jnp.array(__UpperCamelCase    ,			dtype="""i4"""     )    ,			not train    ,			method=_get_features    ,			rngs=__UpperCamelCase    ,			)






							def 				__lowercase	(       self    :      List[str]    ,			snake_case__    :      Any    ,			snake_case__    :      dict = None    ,			snake_case__    :      jax.random.PRNGKey = None    ,			snake_case__    :      Optional[int]=False     ):






												"""simple docstring"""



												__a:    List[str]		     =       {}
												if dropout_rng is not None:
																	__a:    Optional[Any]		     =       dropout_rng

												def _get_features(snake_case__    :      Any    ,			snake_case__    :      int    ,			snake_case__    :      Any     ):
																	__a:    List[Any]		     =       module.vision_model(pixel_values=__UpperCamelCase    ,			deterministic=__UpperCamelCase     )
																	__a:    List[Any]		     =       vision_outputs[1]  # pooled_output
																	__a:    int		     =       module.visual_projection(__UpperCamelCase     )
																	return image_features

												return self.module.apply(
												    {"""params""": params or self.params}    ,			jnp.array(__UpperCamelCase    ,			dtype=jnp.floataa     )    ,			not train    ,			method=_get_features    ,			rngs=__UpperCamelCase    ,			)






							@classmethod
							def 				__lowercase	(       cls    :      str    ,			snake_case__    :      str = None    ,			snake_case__    :      str = None    ,			*snake_case__    :      Tuple    ,			**snake_case__    :      List[Any]    ,			):






												"""simple docstring"""



												__a:    Optional[Any]		     =       {
												    argument[len("""text_"""     ) :]: value for argument, value in kwargs.items() if argument.startswith("""text_"""     )
												}

												__a:    str		     =       {
												    argument[len("""vision_"""     ) :]: value for argument, value in kwargs.items() if argument.startswith("""vision_"""     )
												}

												# remove text, vision kwargs from kwargs
												for key in kwargs_text.keys():
																	del kwargs["text_" + key]
												for key in kwargs_vision.keys():
																	del kwargs["vision_" + key]

												# Load and initialize the text and vision model
												__a:    Tuple		     =       kwargs_text.pop("""model"""    ,			__UpperCamelCase     )
												if text_model is None:
																	assert (
																	    text_model_name_or_path is not None
																	), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
																	from transformers import FlaxAutoModel

																	if "config" not in kwargs_text:
																						from transformers import AutoConfig

																						__a:    str		     =       AutoConfig.from_pretrained(__UpperCamelCase     )
																						__a:    Optional[int]		     =       text_config

																	__a:    Any		     =       FlaxAutoModel.from_pretrained(__UpperCamelCase    ,			*__UpperCamelCase    ,			**__UpperCamelCase     )

												__a:    Optional[int]		     =       kwargs_vision.pop("""model"""    ,			__UpperCamelCase     )
												if vision_model is None:
																	assert (
																	    vision_model_name_or_path is not None
																	), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
																	from transformers import FlaxAutoModel

																	if "config" not in kwargs_vision:
																						from transformers import AutoConfig

																						__a:    Optional[int]		     =       AutoConfig.from_pretrained(__UpperCamelCase     )
																						__a:    Any		     =       vision_config

																	__a:    List[str]		     =       FlaxAutoModel.from_pretrained(__UpperCamelCase    ,			*__UpperCamelCase    ,			**__UpperCamelCase     )

												# instantiate config with corresponding kwargs
												__a:    str		     =       kwargs.pop("""dtype"""    ,			jnp.floataa     )
												__a:    List[Any]		     =       HybridCLIPConfig.from_text_vision_configs(text_model.config    ,			vision_model.config    ,			**__UpperCamelCase     )

												# init model
												__a:    Optional[int]		     =       cls(__UpperCamelCase    ,			*__UpperCamelCase    ,			dtype=__UpperCamelCase    ,			**__UpperCamelCase     )

												if vision_config.model_type == "clip":
																	__a:    Tuple		     =       vision_model.params['''vision_model''']
																	__a:    int		     =       vision_model.params['''visual_projection''']['''kernel''']
												else:
																	__a:    str		     =       vision_model.params

												__a:    Any		     =       text_model.params

												return model





'''simple docstring'''



import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


_lowercase       :						int						=[nn.LayerNorm]

_lowercase       :						Tuple						=logging.get_logger(__name__)

_lowercase       :						List[Any]						=version.parse(version.parse(torch.__version__).base_version)

_lowercase       :						int						=parsed_torch_version_base >= version.parse("2.0")
_lowercase       :						Tuple						=parsed_torch_version_base >= version.parse("1.12")
_lowercase       :						int						=parsed_torch_version_base >= version.parse("1.11")
_lowercase       :						Dict						=parsed_torch_version_base < version.parse("1.11")


def 							__UpperCAmelCase				(				UpperCamelCase__				:Dict   ,				UpperCamelCase__				:List[str]   ,				UpperCamelCase__				:str   ,				UpperCamelCase__				:Any   ,				self				:Optional[int]			)		->						Union[str, Any]:
				from torch import _softmax_backward_data

				if is_torch_less_than_1_11:
								return _softmax_backward_data(UpperCamelCase__   ,				UpperCamelCase__   ,				parent.dim   ,				self			)
				else:
								return _softmax_backward_data(UpperCamelCase__   ,				UpperCamelCase__   ,				parent.dim   ,				self.dtype			)


def 							__UpperCAmelCase				(				UpperCamelCase__				:nn.Linear   ,				UpperCamelCase__				:torch.LongTensor   ,				UpperCamelCase__				:int = 0			)		->						nn.Linear:
				snake_case__      :   Optional[Any]      =    index.to(layer.weight.device			)
				snake_case__      :   int      =    layer.weight.index_select(UpperCamelCase__   ,				UpperCamelCase__			).clone().detach()
				if layer.bias is not None:
								if dim == 1:
												snake_case__      :   Dict      =    layer.bias.clone().detach()
								else:
												snake_case__      :   Optional[Any]      =    layer.bias[index].clone().detach()
				snake_case__      :   Union[str, Any]      =    list(layer.weight.size()			)
				snake_case__      :   int      =    len(UpperCamelCase__			)
				snake_case__      :   int      =    nn.Linear(new_size[1]   ,				new_size[0]   ,				bias=layer.bias is not None			).to(layer.weight.device			)
				snake_case__      :   Union[str, Any]      =    False
				new_layer.weight.copy_(W.contiguous()			)
				snake_case__      :   List[Any]      =    True
				if layer.bias is not None:
								snake_case__      :   Union[str, Any]      =    False
								new_layer.bias.copy_(b.contiguous()			)
								snake_case__      :   Dict      =    True
				return new_layer






class 			_SCREAMING_SNAKE_CASE				(nn.Module      ):



			def __init__(     self		:						int    ,			__UpperCamelCase		:						Tuple    ,			__UpperCamelCase		:						Optional[Any]			)      ->      Union[str, Any]:
							"""simple docstring"""





							super().__init__()
							snake_case__      :   str      =    nf
							snake_case__      :   Tuple      =    nn.Parameter(torch.empty(__UpperCamelCase    ,			__UpperCamelCase			)			)
							snake_case__      :   Optional[Any]      =    nn.Parameter(torch.zeros(__UpperCamelCase			)			)
							nn.init.normal_(self.weight    ,			std=0.02			)







			def 				lowerCAmelCase (     self		:						Dict    ,			__UpperCamelCase		:						Optional[int]			)      ->      Dict:
							"""simple docstring"""





							snake_case__      :   Dict      =    x.size()[:-1] + (self.nf,)
							snake_case__      :   List[str]      =    torch.addmm(self.bias    ,			x.view(-1    ,			x.size(-1			)			)    ,			self.weight			)
							snake_case__      :   Dict      =    x.view(__UpperCamelCase			)
							return x



def 							__UpperCAmelCase				(				UpperCamelCase__				:ConvaD   ,				UpperCamelCase__				:torch.LongTensor   ,				UpperCamelCase__				:int = 1			)		->						ConvaD:
				snake_case__      :   Optional[Any]      =    index.to(layer.weight.device			)
				snake_case__      :   List[Any]      =    layer.weight.index_select(UpperCamelCase__   ,				UpperCamelCase__			).clone().detach()
				if dim == 0:
								snake_case__      :   Any      =    layer.bias.clone().detach()
				else:
								snake_case__      :   Optional[Any]      =    layer.bias[index].clone().detach()
				snake_case__      :   Union[str, Any]      =    list(layer.weight.size()			)
				snake_case__      :   List[str]      =    len(UpperCamelCase__			)
				snake_case__      :   Optional[Any]      =    ConvaD(new_size[1]   ,				new_size[0]			).to(layer.weight.device			)
				snake_case__      :   List[str]      =    False
				new_layer.weight.copy_(W.contiguous()			)
				snake_case__      :   List[Any]      =    True
				snake_case__      :   List[str]      =    False
				new_layer.bias.copy_(b.contiguous()			)
				snake_case__      :   str      =    True
				return new_layer


def 							__UpperCAmelCase				(				UpperCamelCase__				:Union[nn.Linear, ConvaD]   ,				UpperCamelCase__				:torch.LongTensor   ,				UpperCamelCase__				:Optional[int] = None			)		->						Union[nn.Linear, ConvaD]:
				if isinstance(UpperCamelCase__   ,				nn.Linear			):
								return prune_linear_layer(UpperCamelCase__   ,				UpperCamelCase__   ,				dim=0 if dim is None else dim			)
				elif isinstance(UpperCamelCase__   ,				UpperCamelCase__			):
								return prune_convad_layer(UpperCamelCase__   ,				UpperCamelCase__   ,				dim=1 if dim is None else dim			)
				else:
								raise ValueError(F'''Can\'t prune layer of class {layer.__class__}'''			)


def 							__UpperCAmelCase				(				UpperCamelCase__				:Callable[..., torch.Tensor]   ,				UpperCamelCase__				:int   ,				UpperCamelCase__				:int   ,				*UpperCamelCase__				:List[str]			)		->						torch.Tensor:
				assert len(UpperCamelCase__			) > 0, F'''{input_tensors} has to be a tuple/list of tensors'''

				# inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
				snake_case__      :   str      =    len(inspect.signature(UpperCamelCase__			).parameters			)
				if num_args_in_forward_chunk_fn != len(UpperCamelCase__			):
								raise ValueError(
								    F'''forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(UpperCamelCase__			)} input '''
								    '''tensors are given'''			)

				if chunk_size > 0:
								snake_case__      :   Tuple      =    input_tensors[0].shape[chunk_dim]
								for input_tensor in input_tensors:
												if input_tensor.shape[chunk_dim] != tensor_shape:
																raise ValueError(
																    F'''All input tenors have to be of the same shape: {tensor_shape}, '''
																    F'''found shape {input_tensor.shape[chunk_dim]}'''			)

								if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
												raise ValueError(
												    F'''The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk '''
												    F'''size {chunk_size}'''			)

								snake_case__      :   Optional[Any]      =    input_tensors[0].shape[chunk_dim] // chunk_size

								# chunk input tensor into tuples
								snake_case__      :   List[Any]      =    tuple(input_tensor.chunk(UpperCamelCase__   ,				dim=UpperCamelCase__			) for input_tensor in input_tensors			)
								# apply forward fn to every tuple
								snake_case__      :   List[Any]      =    tuple(forward_fn(*UpperCamelCase__			) for input_tensors_chunk in zip(*UpperCamelCase__			)			)
								# concatenate output at same dimension
								return torch.cat(UpperCamelCase__   ,				dim=UpperCamelCase__			)

				return forward_fn(*UpperCamelCase__			)


def 							__UpperCAmelCase				(				UpperCamelCase__				:List[int]   ,				UpperCamelCase__				:int   ,				UpperCamelCase__				:int   ,				UpperCamelCase__				:Set[int]			)		->						Tuple[Set[int], torch.LongTensor]:
				snake_case__      :   str      =    torch.ones(UpperCamelCase__   ,				UpperCamelCase__			)
				snake_case__      :   int      =    set(UpperCamelCase__			) - already_pruned_heads  # Convert to set and remove already pruned heads
				for head in heads:
								# Compute how many pruned heads are before the head and move the index accordingly
								snake_case__      :   str      =    head - sum(1 if h < head else 0 for h in already_pruned_heads			)
								snake_case__      :   Optional[int]      =    0
				snake_case__      :   Optional[Any]      =    mask.view(-1			).contiguous().eq(1			)
				snake_case__      :   torch.LongTensor      =    torch.arange(len(UpperCamelCase__			)			)[mask].long()
				return heads, index


def 							__UpperCAmelCase				(				*UpperCamelCase__				:Union[torch.Tensor, List[torch.Tensor]]   ,				UpperCamelCase__				:Optional[str] = None			)		->						Tuple[torch.Tensor, ...]:
				return torch.meshgrid(*UpperCamelCase__   ,				indexing=UpperCamelCase__			)



def 							__UpperCAmelCase				(				UpperCamelCase__				:torch.Tensor			)		->						Tuple[torch.device, int, int]:
				return tensor.device, storage_ptr(UpperCamelCase__			), storage_size(UpperCamelCase__			)




