'''simple docstring'''

import argparse
import collections

import jax
import jax.numpy as jnp
import torch
import torch.nn as nn
from clip.model import CLIP
from flax.training import checkpoints
from huggingface_hub import Repository

from transformers import (
    CLIPTokenizer,
    OwlViTConfig,
    OwlViTForObjectDetection,
    OwlViTImageProcessor,
    OwlViTModel,
    OwlViTProcessor,
)


_lowerCamelCase :				Optional[Any]														= {
    """vit_b32""": {
        """embed_dim""": 5_12,
        """image_resolution""": 7_68,
        """context_length""": 16,
        """vocab_size""": 4_94_08,
        """vision_layers""": 12,
        """vision_width""": 7_68,
        """vision_patch_size""": 32,
        """transformer_width""": 5_12,
        """transformer_heads""": 8,
        """transformer_layers""": 12,
    },
    """vit_b16""": {
        """embed_dim""": 5_12,
        """image_resolution""": 7_68,
        """context_length""": 16,
        """vocab_size""": 4_94_08,
        """vision_layers""": 12,
        """vision_width""": 7_68,
        """vision_patch_size""": 16,
        """transformer_width""": 5_12,
        """transformer_heads""": 8,
        """transformer_layers""": 12,
    },
    """vit_l14""": {
        """embed_dim""": 7_68,
        """image_resolution""": 8_40,
        """context_length""": 16,
        """vocab_size""": 4_94_08,
        """vision_layers""": 24,
        """vision_width""": 10_24,
        """vision_patch_size""": 14,
        """transformer_width""": 7_68,
        """transformer_heads""": 12,
        """transformer_layers""": 12,
    },
}



def  _lowerCAmelCase				(				__a			,      __a=""			,      __a="/"      )  ->      Any:
     '''simple docstring'''







     _UpperCamelCase  :Dict       =[]

     for k, v in params.items():
          _UpperCamelCase  :int       =parent_key + sep + k if parent_key else k

          if isinstance(__a			,      collections.MutableMapping      ):
               items.extend(flatten_nested_dict(__a			,      __a			,      sep=__a      ).items()      )
          else:
               items.append((new_key, v)      )
     return dict(__a      )



def  _lowerCAmelCase				(				__a      )  ->      Optional[int]:
     '''simple docstring'''







     return jax.tree_util.tree_map(lambda __a      : x.astype(jnp.floataa      ) if x.dtype == jnp.bfloataa else x			,      __a      )



def  _lowerCAmelCase				(				__a			,      __a      )  ->      int:
     '''simple docstring'''







     _UpperCamelCase       ,      _UpperCamelCase       ,      _UpperCamelCase  :int       =pt_attn_layer.in_proj_weight.chunk(3			,      dim=0      )
     _UpperCamelCase       ,      _UpperCamelCase       ,      _UpperCamelCase  :List[Any]       =pt_attn_layer.in_proj_bias.chunk(3			,      dim=0      )

     _UpperCamelCase  :Tuple       =pt_attn_layer.out_proj.weight
     _UpperCamelCase  :Optional[int]       =pt_attn_layer.out_proj.bias

     _UpperCamelCase  :Union[str, Any]       =q_proj
     _UpperCamelCase  :Dict       =q_proj_bias

     _UpperCamelCase  :str       =k_proj
     _UpperCamelCase  :Any       =k_proj_bias

     _UpperCamelCase  :int       =v_proj
     _UpperCamelCase  :Optional[Any]       =v_proj_bias

     _UpperCamelCase  :str       =out_proj_weights
     _UpperCamelCase  :List[str]       =out_proj_bias



def  _lowerCAmelCase				(				__a			,      __a      )  ->      Tuple:
     '''simple docstring'''







     copy_linear(hf_mlp.fca			,      pt_mlp.c_fc      )
     copy_linear(hf_mlp.fca			,      pt_mlp.c_proj      )



def  _lowerCAmelCase				(				__a			,      __a      )  ->      str:
     '''simple docstring'''







     _UpperCamelCase  :List[Any]       =pt_linear.weight
     _UpperCamelCase  :List[Any]       =pt_linear.bias



def  _lowerCAmelCase				(				__a			,      __a      )  ->      Optional[Any]:
     '''simple docstring'''







     copy_linear(hf_layer.layer_norma			,      pt_layer.ln_a      )
     copy_linear(hf_layer.layer_norma			,      pt_layer.ln_a      )

     # copy MLP
     copy_mlp(hf_layer.mlp			,      pt_layer.mlp      )

     # copy attn
     copy_attn_layer(hf_layer.self_attn			,      pt_layer.attn      )



def  _lowerCAmelCase				(				__a			,      __a      )  ->      Tuple:
     '''simple docstring'''







     for hf_layer, pt_layer in zip(__a			,      __a      ):
          copy_layer(__a			,      __a      )



def  _lowerCAmelCase				(				__a			,      __a      )  ->      Union[str, Any]:
     '''simple docstring'''







     _UpperCamelCase  :List[str]       =pt_model.token_embedding.weight
     _UpperCamelCase  :int       =pt_model.positional_embedding

     # copy layer norm
     copy_linear(hf_encoder.final_layer_norm			,      pt_model.ln_final      )

     # copy hidden layers
     copy_layers(hf_encoder.encoder.layers			,      pt_model.transformer.resblocks      )



def  _lowerCAmelCase				(				__a			,      __a      )  ->      List[str]:
     '''simple docstring'''







     _UpperCamelCase  :List[str]       =pt_model.text_projection.data.T

     # copy text encoder
     copy_encoder(hf_model.text_model			,      __a      )



def  _lowerCAmelCase				(				__a			,      __a      )  ->      Union[str, Any]:
     '''simple docstring'''







     _UpperCamelCase  :int       =pt_model.visual.proj.data.T

     # copy layer norms
     copy_linear(hf_model.vision_model.pre_layernorm			,      pt_model.visual.ln_pre      )
     copy_linear(hf_model.vision_model.post_layernorm			,      pt_model.visual.ln_post      )

     # copy embeds
     _UpperCamelCase  :Dict       =pt_model.visual.conva.weight.data
     _UpperCamelCase  :Tuple       =pt_model.visual.class_embedding
     _UpperCamelCase  :Tuple       =pt_model.visual.positional_embedding.data

     # copy encoder
     copy_layers(hf_model.vision_model.encoder.layers			,      pt_model.visual.transformer.resblocks      )



def  _lowerCAmelCase				(				__a			,      __a      )  ->      List[Any]:
     '''simple docstring'''







     _UpperCamelCase  :List[str]       =flatten_nested_dict(flax_params["""backbone"""]["""merged_class_token"""]      )

     _UpperCamelCase  :List[Any]       =torch.from_numpy(flax_class_token_params["""scale"""]      )
     _UpperCamelCase  :Optional[Any]       =torch.from_numpy(flax_class_token_params["""bias"""]      )
     _UpperCamelCase  :str       =nn.Parameter(__a      )
     _UpperCamelCase  :Optional[Any]       =nn.Parameter(__a      )



def  _lowerCAmelCase				(				__a			,      __a      )  ->      Dict:
     '''simple docstring'''







     _UpperCamelCase  :Any       =hf_model.state_dict()
     _UpperCamelCase  :List[Any]       ={}

     # Rename class prediction head flax params to pytorch HF
     _UpperCamelCase  :Any       =flatten_nested_dict(flax_params["""class_head"""]      )

     for flax_key, v in flax_class_params.items():
          _UpperCamelCase  :int       =flax_key.replace("""/"""			,      """."""      )
          _UpperCamelCase  :Union[str, Any]       =torch_key.replace(""".kernel"""			,      """.weight"""      )
          _UpperCamelCase  :Dict       =torch_key.replace("""Dense_0"""			,      """dense0"""      )
          _UpperCamelCase  :Optional[Any]       ="""class_head.""" + torch_key

          if "weight" in torch_key and v.ndim == 2:
               _UpperCamelCase  :Tuple       =v.T

          _UpperCamelCase  :Optional[Any]       =nn.Parameter(torch.from_numpy(__a      )      )

     # Rename box prediction box flax params to pytorch HF
     _UpperCamelCase  :Optional[Any]       =flatten_nested_dict(flax_params["""obj_box_head"""]      )

     for flax_key, v in flax_box_params.items():
          _UpperCamelCase  :str       =flax_key.replace("""/"""			,      """."""      )
          _UpperCamelCase  :Union[str, Any]       =torch_key.replace(""".kernel"""			,      """.weight"""      )
          _UpperCamelCase  :Optional[int]       =torch_key.replace("""_"""			,      """"""      ).lower()
          _UpperCamelCase  :List[Any]       ="""box_head.""" + torch_key

          if "weight" in torch_key and v.ndim == 2:
               _UpperCamelCase  :str       =v.T

          _UpperCamelCase  :List[str]       =nn.Parameter(torch.from_numpy(__a      )      )

     # Copy flax params to PyTorch params
     for name, param in new_params.items():
          if name in pt_params.keys():
               pt_params[name].copy_(__a      )



def  _lowerCAmelCase				(				__a			,      __a      )  ->      Any:
     '''simple docstring'''







     for k, v in flax_attn_params.items():
          if k.startswith("""transformer"""      ):
               _UpperCamelCase  :int       =k.replace("""transformer.resblocks"""			,      """text_model.encoder.layers"""      )
          else:
               _UpperCamelCase  :Optional[Any]       =k.replace("""visual.transformer.resblocks"""			,      """vision_model.encoder.layers"""      )

          _UpperCamelCase  :str       =torch_key.replace("""attn"""			,      """self_attn"""      )
          _UpperCamelCase  :int       =torch_key.replace("""key"""			,      """k_proj"""      )
          _UpperCamelCase  :List[Any]       =torch_key.replace("""value"""			,      """v_proj"""      )
          _UpperCamelCase  :str       =torch_key.replace("""query"""			,      """q_proj"""      )
          _UpperCamelCase  :List[str]       =torch_key.replace("""out"""			,      """out_proj"""      )

          if "bias" in torch_key and v.ndim == 2:
               _UpperCamelCase  :Dict       =v.shape[0] * v.shape[1]
               _UpperCamelCase  :Union[str, Any]       =v.reshape(__a      )

          if "weight" in torch_key and "out" in torch_key:
               _UpperCamelCase  :Union[str, Any]       =(v.shape[0] * v.shape[1], v.shape[2])
               _UpperCamelCase  :Optional[Any]       =v.reshape(__a      ).T

          if "weight" in torch_key and "out" not in torch_key:
               _UpperCamelCase  :Tuple       =(v.shape[0], v.shape[1] * v.shape[2])
               _UpperCamelCase  :List[Any]       =v.reshape(__a      ).T

          # Copy flax CLIP attn params to HF PyTorch params
          _UpperCamelCase  :List[str]       =torch.from_numpy(__a      )
          hf_backbone.state_dict()[torch_key].copy_(__a      )



def  _lowerCAmelCase				(				__a      )  ->      Tuple:
     '''simple docstring'''







     _UpperCamelCase  :Optional[int]       ={}
     _UpperCamelCase  :Any       =[]

     for k, v in params.items():
          if "attn." in k:
               _UpperCamelCase  :Optional[Any]       =k[: k.rindex("""attn."""      ) + 5]
               if base in processed_attn_layers:
                    continue

               processed_attn_layers.append(__a      )
               _UpperCamelCase  :Optional[int]       =params[base + """out.weight"""].shape[-1]
               _UpperCamelCase  :Optional[Any]       =params[base + """out.weight"""].reshape(__a			,      __a      ).T
               _UpperCamelCase  :Tuple       =params[base + """out.bias"""]
          else:
               _UpperCamelCase  :Tuple       =v
     return new_params



def  _lowerCAmelCase				(				__a			,      __a      )  ->      Dict:
     '''simple docstring'''







     _UpperCamelCase  :Optional[int]       =CLIP(**__a      )
     torch_model.eval()
     _UpperCamelCase  :str       =torch_model.state_dict()

     _UpperCamelCase  :Optional[Any]       =flatten_nested_dict(flax_params["""backbone"""]["""clip"""]      )
     _UpperCamelCase  :int       ={}

     for flax_key, v in flax_clip_params.items():
          _UpperCamelCase  :Union[str, Any]       =flax_key.replace("""/"""			,      """."""      )
          _UpperCamelCase  :str       =torch_key.replace("""text.token_embedding.embedding"""			,      """token_embedding.kernel"""      )

          if (
              torch_key.startswith("""text.transformer"""      )
              or torch_key.startswith("""text.text_projection"""      )
              or torch_key.startswith("""text.ln_final"""      )
              or torch_key.startswith("""text.positional_embedding"""      )
          ):
               _UpperCamelCase  :List[str]       =torch_key[5:]

          _UpperCamelCase  :List[Any]       =torch_key.replace("""text_projection.kernel"""			,      """text_projection"""      )
          _UpperCamelCase  :Dict       =torch_key.replace("""visual.proj.kernel"""			,      """visual.proj"""      )
          _UpperCamelCase  :Tuple       =torch_key.replace(""".scale"""			,      """.weight"""      )
          _UpperCamelCase  :List[Any]       =torch_key.replace(""".kernel"""			,      """.weight"""      )

          if "conv" in torch_key or "downsample.0.weight" in torch_key:
               _UpperCamelCase  :List[str]       =v.transpose(3			,      2			,      0			,      1      )

          elif "weight" in torch_key and v.ndim == 2 and "embedding" not in torch_key:
               # Fully connected layers are transposed, embeddings are not
               _UpperCamelCase  :List[Any]       =v.T

          _UpperCamelCase  :Union[str, Any]       =v

     _UpperCamelCase  :List[str]       =_convert_attn_layers(__a      )
     new_torch_params.update(__a      )
     _UpperCamelCase  :Dict       ={}

     # Copy flax CLIP backbone params to PyTorch params
     for name, param in new_torch_params.items():
          if name in torch_clip_params.keys():
               _UpperCamelCase  :List[str]       =torch.from_numpy(new_torch_params[name]      )
               torch_clip_params[name].copy_(__a      )
          else:
               _UpperCamelCase  :Optional[int]       =param

     return torch_clip_params, torch_model, attn_params



@torch.no_grad()
def  _lowerCAmelCase				(				__a			,      __a			,      __a			,      __a			,      __a=None      )  ->      Any:
     '''simple docstring'''







     _UpperCamelCase  :Union[str, Any]       =Repository(__a			,      clone_from=F'''google/{pytorch_dump_folder_path}'''      )
     repo.git_pull()

     if config_path is not None:
          _UpperCamelCase  :List[Any]       =OwlViTConfig.from_pretrained(__a      )
     else:
          _UpperCamelCase  :Any       =OwlViTConfig()

     _UpperCamelCase  :Optional[Any]       =OwlViTModel(__a      ).eval()
     _UpperCamelCase  :List[str]       =OwlViTForObjectDetection(__a      ).eval()

     copy_text_model_and_projection(__a			,      __a      )
     copy_vision_model_and_projection(__a			,      __a      )
     _UpperCamelCase  :int       =pt_backbone.logit_scale
     copy_flax_attn_params(__a			,      __a      )

     _UpperCamelCase  :Optional[int]       =hf_backbone
     copy_class_merge_token(__a			,      __a      )
     copy_class_box_heads(__a			,      __a      )

     # Save HF model
     hf_model.save_pretrained(repo.local_dir      )

     # Initialize image processor
     _UpperCamelCase  :List[Any]       =OwlViTImageProcessor(
         size=config.vision_config.image_size			,      crop_size=config.vision_config.image_size      )
     # Initialize tokenizer
     _UpperCamelCase  :List[Any]       =CLIPTokenizer.from_pretrained("""openai/clip-vit-base-patch32"""			,      pad_token="""!"""			,      model_max_length=16      )

     # Initialize processor
     _UpperCamelCase  :Any       =OwlViTProcessor(image_processor=__a			,      tokenizer=__a      )
     image_processor.save_pretrained(repo.local_dir      )
     processor.save_pretrained(repo.local_dir      )

     repo.git_add()
     repo.git_commit("""Upload model and processor"""      )
     repo.git_push()


if __name__ == "__main__":
      _lowerCamelCase :				Dict														= argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--owlvit_version""",
          default=None,
          type=str,
          required=True,
          help="""OWL-ViT model name [clip_b16, clip_b32, clip_l14].""",
      )
      parser.add_argument(
          """--owlvit_checkpoint""", default=None, type=str, required=True, help="""Path to flax model checkpoint."""
      )
      parser.add_argument("""--hf_config""", default=None, type=str, required=True, help="""Path to HF model config.""")
      parser.add_argument(
          """--pytorch_dump_folder_path""", default="""hf_model""", type=str, help="""Path to the output PyTorch model."""
      )
      _lowerCamelCase :				int														= parser.parse_args()

      # Initialize PyToch clip model
      _lowerCamelCase :				Any														= args.owlvit_version
      if model_name == "clip_b16":
            _lowerCamelCase :				Tuple														= CONFIGS["""vit_b16"""]
      elif model_name == "clip_b32":
            _lowerCamelCase :				int														= CONFIGS["""vit_b32"""]
      elif model_name == "clip_l14":
            _lowerCamelCase :				Optional[int]														= CONFIGS["""vit_l14"""]

      # Load from checkpoint and convert params to float-32
      _lowerCamelCase :				Optional[int]														= checkpoints.restore_checkpoint(args.owlvit_checkpoint, target=None)["""optimizer"""]["""target"""]
      _lowerCamelCase :				Any														= jax.tree_util.tree_map(lambda x: x.astype(jnp.floataa) if x.dtype == jnp.bfloataa else x, variables)
      del variables

      # Convert CLIP backbone
      _lowerCamelCase							,    _lowerCamelCase							,    _lowerCamelCase :				Union[str, Any]														= convert_clip_backbone(flax_params, torch_config)

      convert_owlvit_checkpoint(clip_pt, flax_params, attn_params, args.pytorch_dump_folder_path, args.hf_config)





import math
from typing import Optional

import torch
import torch.utils.checkpoint as checkpoint
from torch import nn

from ....modeling_utils import PreTrainedModel
from ....utils import add_start_docstrings, logging
from ...bert.modeling_bert import BertModel
from .configuration_retribert import RetriBertConfig


a_ 		=  logging.get_logger(__name__)

a_ 		=  [
    "yjernite/retribert-base-uncased",
    # See all RetriBert models at https://huggingface.co/models?filter=retribert
]

class 			_lowerCamelCase    (      _SCREAMING_SNAKE_CASE ):

      """simple docstring"""

      lowerCAmelCase__      :			Optional[Any]     =       RetriBertConfig
      lowerCAmelCase__      :			Optional[Any]     =       None
      lowerCAmelCase__      :			Dict     =       "retribert"







      def    snake_case						(  self       :			Dict			,							snake_case       :			Optional[Any]					):
          if isinstance(snake_case			,							nn.Linear					):
              module.weight.data.normal_(mean=0.0			,							std=self.config.initializer_range					)
              if module.bias is not None:
                  module.bias.data.zero_()
          elif isinstance(snake_case			,							nn.Embedding					):
              module.weight.data.normal_(mean=0.0			,							std=self.config.initializer_range					)
              if module.padding_idx is not None:
                  module.weight.data[module.padding_idx].zero_()
          elif isinstance(snake_case			,							nn.LayerNorm					):
              module.bias.data.zero_()
              module.weight.data.fill_(1.0					)







a_ 		=  R"\n\n    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the\n    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads\n    etc.)\n\n    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.\n    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage\n    and behavior.\n\n    Parameters:\n        config ([`RetriBertConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n"

@add_start_docstrings(
    "Bert Based model to embed queries or document for document retrieval."				,      _SCREAMING_SNAKE_CASE				,      )
class 			_lowerCamelCase    (      _SCREAMING_SNAKE_CASE ):

      """simple docstring"""

      def __init__(  self       :			List[Any]			,							snake_case       :			RetriBertConfig					):
          super().__init__(snake_case					)
          __UpperCamelCase			=    config.projection_dim

          __UpperCamelCase			=    BertModel(snake_case					)
          __UpperCamelCase			=    None if config.share_encoders else BertModel(snake_case					)
          __UpperCamelCase			=    nn.Dropout(config.hidden_dropout_prob					)
          __UpperCamelCase			=    nn.Linear(config.hidden_size			,							config.projection_dim			,							bias=snake_case					)
          __UpperCamelCase			=    nn.Linear(config.hidden_size			,							config.projection_dim			,							bias=snake_case					)

          __UpperCamelCase			=    nn.CrossEntropyLoss(reduction='''mean'''					)

          # Initialize weights and apply final processing
          self.post_init()







      def    snake_case						(  self       :			Dict			,							snake_case       :			Dict			,							snake_case       :			List[str]			,							snake_case       :			Dict			,							snake_case       :			Any=-1			,							):
          # reproduces BERT forward pass with checkpointing
          if checkpoint_batch_size < 0 or input_ids.shape[0] < checkpoint_batch_size:
              return sent_encoder(snake_case			,							attention_mask=snake_case					)[1]
          else:
              # prepare implicit variables
              __UpperCamelCase			=    input_ids.device
              __UpperCamelCase			=    input_ids.size()
              __UpperCamelCase			=    torch.zeros(snake_case			,							dtype=torch.long			,							device=snake_case					)
              __UpperCamelCase			=    [None] * sent_encoder.config.num_hidden_layers
              __UpperCamelCase			=    sent_encoder.get_extended_attention_mask(
                  snake_case			,							snake_case					)

              # define function for checkpointing
              def partial_encode(*snake_case       :			Optional[Any]					):
                  __UpperCamelCase			=    sent_encoder.encoder(
                      inputs[0]			,							attention_mask=inputs[1]			,							head_mask=snake_case			,							)
                  __UpperCamelCase			=    encoder_outputs[0]
                  __UpperCamelCase			=    sent_encoder.pooler(snake_case					)
                  return pooled_output

              # run embedding layer on everything at once
              __UpperCamelCase			=    sent_encoder.embeddings(
                  input_ids=snake_case			,							position_ids=snake_case			,							token_type_ids=snake_case			,							inputs_embeds=snake_case					)
              # run encoding and pooling on one mini-batch at a time
              __UpperCamelCase			=    []
              for b in range(math.ceil(input_ids.shape[0] / checkpoint_batch_size					)					):
                  __UpperCamelCase			=    embedding_output[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
                  __UpperCamelCase			=    extended_attention_mask[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
                  __UpperCamelCase			=    checkpoint.checkpoint(snake_case			,							snake_case			,							snake_case					)
                  pooled_output_list.append(snake_case					)
              return torch.cat(snake_case			,							dim=0					)







      def    snake_case						(  self       :			Dict			,							snake_case       :			Any			,							snake_case       :			Union[str, Any]=None			,							snake_case       :			List[str]=-1			,							):
          __UpperCamelCase			=    self.embed_sentences_checkpointed(
              snake_case			,							snake_case			,							self.bert_query			,							snake_case			,							)
          return self.project_query(snake_case					)







      def    snake_case						(  self       :			Optional[Any]			,							snake_case       :			Optional[Any]			,							snake_case       :			Optional[int]=None			,							snake_case       :			Optional[Any]=-1			,							):
          __UpperCamelCase			=    self.embed_sentences_checkpointed(
              snake_case			,							snake_case			,							self.bert_query if self.bert_doc is None else self.bert_doc			,							snake_case			,							)
          return self.project_doc(snake_case					)







      def    snake_case						(  self       :			List[str]			,							snake_case       :			torch.LongTensor			,							snake_case       :			Optional[torch.FloatTensor]			,							snake_case       :			torch.LongTensor			,							snake_case       :			Optional[torch.FloatTensor]			,							snake_case       :			int = -1			,							):
          __UpperCamelCase			=    input_ids_query.device
          __UpperCamelCase			=    self.embed_questions(snake_case			,							snake_case			,							snake_case					)
          __UpperCamelCase			=    self.embed_answers(snake_case			,							snake_case			,							snake_case					)
          __UpperCamelCase			=    torch.mm(snake_case			,							a_reps.t()					)
          __UpperCamelCase			=    self.ce_loss(snake_case			,							torch.arange(compare_scores.shape[1]					).to(snake_case					)					)
          __UpperCamelCase			=    self.ce_loss(compare_scores.t()			,							torch.arange(compare_scores.shape[0]					).to(snake_case					)					)
          __UpperCamelCase			=    (loss_qa + loss_aq) / 2
          return loss






