






"""simple docstring"""


import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           RemBertConfig,
           RemBertForCausalLM,
           RemBertForMaskedLM,
           RemBertForMultipleChoice,
           RemBertForQuestionAnswering,
           RemBertForSequenceClassification,
           RemBertForTokenClassification,
           RemBertModel,
       )
       from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST


class    snake_case						:


   '''simple docstring'''

   def __init__(      self					:	Any,			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	int=13,			_lowerCamelCase					:	int=7,			_lowerCamelCase					:	Union[str, Any]=True,			_lowerCamelCase					:	int=True,			_lowerCamelCase					:	Optional[int]=True,			_lowerCamelCase					:	str=True,			_lowerCamelCase					:	Optional[Any]=99,			_lowerCamelCase					:	Optional[int]=32,			_lowerCamelCase					:	Optional[Any]=18,			_lowerCamelCase					:	List[str]=43,			_lowerCamelCase					:	int=5,			_lowerCamelCase					:	List[Any]=4,			_lowerCamelCase					:	List[Any]=37,			_lowerCamelCase					:	Union[str, Any]="gelu",			_lowerCamelCase					:	List[str]=0.1,			_lowerCamelCase					:	Optional[int]=0.1,			_lowerCamelCase					:	Optional[Any]=5_12,			_lowerCamelCase					:	List[str]=16,			_lowerCamelCase					:	Optional[Any]=2,			_lowerCamelCase					:	str=0.02,			_lowerCamelCase					:	int=3,			_lowerCamelCase					:	str=4,			_lowerCamelCase					:	Dict=None,			):

     '''simple docstring'''
     __A		     =					parent
     __A		     =					batch_size
     __A		     =					seq_length
     __A		     =					is_training
     __A		     =					use_input_mask
     __A		     =					use_token_type_ids
     __A		     =					use_labels
     __A		     =					vocab_size
     __A		     =					hidden_size
     __A		     =					input_embedding_size
     __A		     =					output_embedding_size
     __A		     =					num_hidden_layers
     __A		     =					num_attention_heads
     __A		     =					intermediate_size
     __A		     =					hidden_act
     __A		     =					hidden_dropout_prob
     __A		     =					attention_probs_dropout_prob
     __A		     =					max_position_embeddings
     __A		     =					type_vocab_size
     __A		     =					type_sequence_label_size
     __A		     =					initializer_range
     __A		     =					num_labels
     __A		     =					num_choices
     __A		     =					scope


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[str]    ):

     '''simple docstring'''
     __A		     =					ids_tensor([self.batch_size, self.seq_length],			self.vocab_size    )

     __A		     =					None
     if self.use_input_mask:
       __A		     =					random_attention_mask([self.batch_size, self.seq_length]    )

     __A		     =					None
     if self.use_token_type_ids:
       __A		     =					ids_tensor([self.batch_size, self.seq_length],			self.type_vocab_size    )

     __A		     =					None
     __A		     =					None
     __A		     =					None
     if self.use_labels:
       __A		     =					ids_tensor([self.batch_size],			self.type_sequence_label_size    )
       __A		     =					ids_tensor([self.batch_size, self.seq_length],			self.num_labels    )
       __A		     =					ids_tensor([self.batch_size],			self.num_choices    )

     __A		     =					RemBertConfig(
         vocab_size=self.vocab_size,			hidden_size=self.hidden_size,			input_embedding_size=self.input_embedding_size,			output_embedding_size=self.output_embedding_size,			num_hidden_layers=self.num_hidden_layers,			num_attention_heads=self.num_attention_heads,			intermediate_size=self.intermediate_size,			hidden_act=self.hidden_act,			hidden_dropout_prob=self.hidden_dropout_prob,			attention_probs_dropout_prob=self.attention_probs_dropout_prob,			max_position_embeddings=self.max_position_embeddings,			type_vocab_size=self.type_vocab_size,			is_decoder=__snake_case,			initializer_range=self.initializer_range,			)

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int]    ):

     '''simple docstring'''
     (
         __A		
     )		     =					self.prepare_config_and_inputs()

     __A		     =					True
     __A		     =					floats_tensor([self.batch_size, self.seq_length, self.hidden_size]    )
     __A		     =					ids_tensor([self.batch_size, self.seq_length],			vocab_size=2    )

     return (
         config,
         input_ids,
         token_type_ids,
         input_mask,
         sequence_labels,
         token_labels,
         choice_labels,
         encoder_hidden_states,
         encoder_attention_mask,
     )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int],			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	Union[str, Any]    ):

     '''simple docstring'''
     __A		     =					RemBertModel(config=__snake_case    )
     model.to(__snake_case    )
     model.eval()
     __A		     =					model(__snake_case,			attention_mask=__snake_case,			token_type_ids=__snake_case    )
     __A		     =					model(__snake_case,			token_type_ids=__snake_case    )
     __A		     =					model(__snake_case    )
     self.parent.assertEqual(result.last_hidden_state.shape,			(self.batch_size, self.seq_length, self.hidden_size)    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	str,			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	str,			_lowerCamelCase					:	int,			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	str,			):

     '''simple docstring'''
     __A		     =					True
     __A		     =					RemBertModel(__snake_case    )
     model.to(__snake_case    )
     model.eval()
     __A		     =					model(
         __snake_case,			attention_mask=__snake_case,			token_type_ids=__snake_case,			encoder_hidden_states=__snake_case,			encoder_attention_mask=__snake_case,			)
     __A		     =					model(
         __snake_case,			attention_mask=__snake_case,			token_type_ids=__snake_case,			encoder_hidden_states=__snake_case,			)
     __A		     =					model(__snake_case,			attention_mask=__snake_case,			token_type_ids=__snake_case    )
     self.parent.assertEqual(result.last_hidden_state.shape,			(self.batch_size, self.seq_length, self.hidden_size)    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple,			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Optional[int],			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	Optional[int],			_lowerCamelCase					:	str,			):

     '''simple docstring'''
     __A		     =					RemBertForCausalLM(config=__snake_case    )
     model.to(__snake_case    )
     model.eval()
     __A		     =					model(__snake_case,			attention_mask=__snake_case,			token_type_ids=__snake_case,			labels=__snake_case    )
     self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.seq_length, self.vocab_size)    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[str],			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Optional[int],			_lowerCamelCase					:	Optional[int],			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	str,			_lowerCamelCase					:	Dict    ):

     '''simple docstring'''
     __A		     =					RemBertForMaskedLM(config=__snake_case    )
     model.to(__snake_case    )
     model.eval()
     __A		     =					model(__snake_case,			attention_mask=__snake_case,			token_type_ids=__snake_case,			labels=__snake_case    )
     self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.seq_length, self.vocab_size)    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any],			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	Any,			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	Union[str, Any],			):

     '''simple docstring'''
     __A		     =					True
     __A		     =					True
     __A		     =					RemBertForCausalLM(config=__snake_case    )
     model.to(__snake_case    )
     model.eval()

     # first forward pass
     __A		     =					model(
         __snake_case,			attention_mask=__snake_case,			encoder_hidden_states=__snake_case,			encoder_attention_mask=__snake_case,			use_cache=__snake_case,			)
     __A		     =					outputs.past_key_values

     # create hypothetical multiple next token and extent to next_input_ids
     __A		     =					ids_tensor((self.batch_size, 3),			config.vocab_size    )
     __A		     =					ids_tensor((self.batch_size, 3),			vocab_size=2    )

     # append to next input_ids and
     __A		     =					torch.cat([input_ids, next_tokens],			dim=-1    )
     __A		     =					torch.cat([input_mask, next_mask],			dim=-1    )

     __A		     =					model(
         __snake_case,			attention_mask=__snake_case,			encoder_hidden_states=__snake_case,			encoder_attention_mask=__snake_case,			output_hidden_states=__snake_case,			)['hidden_states'][0]
     __A		     =					model(
         __snake_case,			attention_mask=__snake_case,			encoder_hidden_states=__snake_case,			encoder_attention_mask=__snake_case,			past_key_values=__snake_case,			output_hidden_states=__snake_case,			)['hidden_states'][0]

     # select random slice
     __A		     =					ids_tensor((1,),			output_from_past.shape[-1]    ).item()
     __A		     =					output_from_no_past[:, -3:, random_slice_idx].detach()
     __A		     =					output_from_past[:, :, random_slice_idx].detach()

     self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]    )

     # test that outputs are equal for slice
     self.parent.assertTrue(torch.allclose(__snake_case,			__snake_case,			atol=1e-3    )    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int],			_lowerCamelCase					:	Any,			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	str,			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	List[Any]    ):

     '''simple docstring'''
     __A		     =					RemBertForQuestionAnswering(config=__snake_case    )
     model.to(__snake_case    )
     model.eval()
     __A		     =					model(
         __snake_case,			attention_mask=__snake_case,			token_type_ids=__snake_case,			start_positions=__snake_case,			end_positions=__snake_case,			)
     self.parent.assertEqual(result.start_logits.shape,			(self.batch_size, self.seq_length)    )
     self.parent.assertEqual(result.end_logits.shape,			(self.batch_size, self.seq_length)    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any],			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	Optional[Any]    ):

     '''simple docstring'''
     __A		     =					self.num_labels
     __A		     =					RemBertForSequenceClassification(__snake_case    )
     model.to(__snake_case    )
     model.eval()
     __A		     =					model(__snake_case,			attention_mask=__snake_case,			token_type_ids=__snake_case,			labels=__snake_case    )
     self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.num_labels)    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int],			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	Any,			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	Optional[int],			_lowerCamelCase					:	Union[str, Any]    ):

     '''simple docstring'''
     __A		     =					self.num_labels
     __A		     =					RemBertForTokenClassification(config=__snake_case    )
     model.to(__snake_case    )
     model.eval()
     __A		     =					model(__snake_case,			attention_mask=__snake_case,			token_type_ids=__snake_case,			labels=__snake_case    )
     self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.seq_length, self.num_labels)    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	str,			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	str,			_lowerCamelCase					:	Optional[int],			_lowerCamelCase					:	List[Any]    ):

     '''simple docstring'''
     __A		     =					self.num_choices
     __A		     =					RemBertForMultipleChoice(config=__snake_case    )
     model.to(__snake_case    )
     model.eval()
     __A		     =					input_ids.unsqueeze(1    ).expand(-1,			self.num_choices,			-1    ).contiguous()
     __A		     =					token_type_ids.unsqueeze(1    ).expand(-1,			self.num_choices,			-1    ).contiguous()
     __A		     =					input_mask.unsqueeze(1    ).expand(-1,			self.num_choices,			-1    ).contiguous()
     __A		     =					model(
         __snake_case,			attention_mask=__snake_case,			token_type_ids=__snake_case,			labels=__snake_case,			)
     self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.num_choices)    )






   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any]    ):

     '''simple docstring'''
     __A		     =					self.prepare_config_and_inputs()
     (
         __A		
     )		     =					config_and_inputs
     __A		     =					{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
     return config, inputs_dict





@require_torch
class    snake_case						(				_lowercase  ,      _lowercase  ,      unittest.TestCase		):


   '''simple docstring'''

   A_		:      str							=					(
       (
           RemBertModel,
           RemBertForMaskedLM,
           RemBertForCausalLM,
           RemBertForMultipleChoice,
           RemBertForQuestionAnswering,
           RemBertForSequenceClassification,
           RemBertForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   A_		:      Dict							=					(RemBertForCausalLM,) if is_torch_available() else ()
   A_		:      List[Any]							=					(
       {
           "feature-extraction": RemBertModel,
           "fill-mask": RemBertForMaskedLM,
           "question-answering": RemBertForQuestionAnswering,
           "text-classification": RemBertForSequenceClassification,
           "text-generation": RemBertForCausalLM,
           "token-classification": RemBertForTokenClassification,
           "zero-shot": RemBertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any]    ):

     '''simple docstring'''
     __A		     =					RemBertModelTester(self    )
     __A		     =					ConfigTester(self,			config_class=__snake_case,			hidden_size=37    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     self.config_tester.run_common_tests()


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*__snake_case    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     for type in ["absolute", "relative_key", "relative_key_query"]:
       __A		     =					type
       self.model_tester.create_and_check_model(*__snake_case    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[Any]    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*__snake_case    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*__snake_case    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs_for_decoder()
     self.model_tester.create_and_check_decoder_model_past_large_inputs(*__snake_case    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*__snake_case    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[str]    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*__snake_case    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*__snake_case    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int]    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs_for_decoder()
     self.model_tester.create_and_check_model_as_decoder(*__snake_case    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any]    ):

     '''simple docstring'''
     # This regression test was failing with PyTorch < 1.3
     (
         __A		
     )		     =					self.model_tester.prepare_config_and_inputs_for_decoder()

     __A		     =					None

     self.model_tester.create_and_check_model_as_decoder(
         __snake_case,			__snake_case,			__snake_case,			__snake_case,			__snake_case,			__snake_case,			__snake_case,			__snake_case,			__snake_case,			)






   @slow
   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int]    ):

     '''simple docstring'''
     for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       __A		     =					RemBertModel.from_pretrained(__snake_case    )
       self.assertIsNotNone(__snake_case    )





@require_torch
class    snake_case						(				unittest.TestCase		):


   '''simple docstring'''

   @slow
   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any    ):

     '''simple docstring'''
     # Test exact values at the last hidden layer
     __A		     =					RemBertModel.from_pretrained('''google/rembert'''    )
     __A		     =					torch.tensor([[3_12, 5_64_98, 3_13, 21_25, 3_13]]    )
     __A		     =					torch.tensor([[0, 0, 0, 1, 1]]    )
     with torch.no_grad():
       __A		     =					model(__snake_case,			token_type_ids=__snake_case,			output_hidden_states=__snake_case    )

     __A		     =					11_52

     __A		     =					torch.Size((1, 5, hidden_size)    )
     self.assertEqual(output['''last_hidden_state'''].shape,			__snake_case    )

     __A		     =					torch.tensor(
         [
             [
                 [0.07_54, -0.20_22, 0.19_04],
                 [-0.33_54, -0.36_92, -0.47_91],
                 [-0.23_14, -0.67_29, -0.07_49],
                 [-0.03_96, -0.31_05, -0.42_34],
                 [-0.15_71, -0.05_25, 0.53_53],
             ]
         ]    )

     # Running on the original tf implementation gives slightly different results here.
     # Not clear why this variations is present
     # TODO: Find reason for discrepancy
     # expected_original_implementation = [[
     #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
     #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
     #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
     #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
     #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
     # ]]

     self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3],			__snake_case,			atol=1e-4    )    )







import argparse

import torch
from clip import load

from transformers import CLIPConfig, CLIPModel






def        lowerCamelCase_  (				UpperCAmelCase_				:  Optional[int]				,							UpperCAmelCase_				:  Tuple						)    ->					str:




	'''simple docstring'''




	_UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase :		Tuple								=    pt_attn_layer.in_proj_weight.chunk(3				,							dim=0						)
	_UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase :		str								=    pt_attn_layer.in_proj_bias.chunk(3				,							dim=0						)

	_UpperCamelCase :		Dict								=    pt_attn_layer.out_proj.weight
	_UpperCamelCase :		Tuple								=    pt_attn_layer.out_proj.bias

	_UpperCamelCase :		Tuple								=    q_proj
	_UpperCamelCase :		Dict								=    q_proj_bias

	_UpperCamelCase :		Dict								=    k_proj
	_UpperCamelCase :		str								=    k_proj_bias

	_UpperCamelCase :		Dict								=    v_proj
	_UpperCamelCase :		Optional[int]								=    v_proj_bias

	_UpperCamelCase :		Any								=    out_proj_weights
	_UpperCamelCase :		Optional[int]								=    out_proj_bias






def        lowerCamelCase_  (				UpperCAmelCase_				:  Optional[int]				,							UpperCAmelCase_				:  Union[str, Any]						)    ->					List[Any]:




	'''simple docstring'''




	copy_linear(hf_mlp.fca				,							pt_mlp.c_fc						)
	copy_linear(hf_mlp.fca				,							pt_mlp.c_proj						)






def        lowerCamelCase_  (				UpperCAmelCase_				:  List[str]				,							UpperCAmelCase_				:  List[str]						)    ->					List[Any]:




	'''simple docstring'''




	_UpperCamelCase :		int								=    pt_linear.weight
	_UpperCamelCase :		int								=    pt_linear.bias






def        lowerCamelCase_  (				UpperCAmelCase_				:  str				,							UpperCAmelCase_				:  Union[str, Any]						)    ->					Dict:




	'''simple docstring'''




	copy_linear(hf_layer.layer_norma				,							pt_layer.ln_a						)
	copy_linear(hf_layer.layer_norma				,							pt_layer.ln_a						)

	# copy MLP
	copy_mlp(hf_layer.mlp				,							pt_layer.mlp						)

	# copy attn
	copy_attn_layer(hf_layer.self_attn				,							pt_layer.attn						)






def        lowerCamelCase_  (				UpperCAmelCase_				:  Optional[Any]				,							UpperCAmelCase_				:  List[str]						)    ->					Dict:




	'''simple docstring'''




	for hf_layer, pt_layer in zip(UpperCAmelCase_				,							UpperCAmelCase_						):
		copy_layer(UpperCAmelCase_				,							UpperCAmelCase_						)






def        lowerCamelCase_  (				UpperCAmelCase_				:  Any				,							UpperCAmelCase_				:  Optional[Any]						)    ->					int:




	'''simple docstring'''




	_UpperCamelCase :		Optional[int]								=    pt_model.token_embedding.weight
	_UpperCamelCase :		Tuple								=    pt_model.positional_embedding

	# copy layer norm
	copy_linear(hf_encoder.final_layer_norm				,							pt_model.ln_final						)

	# copy hidden layers
	copy_layers(hf_encoder.encoder.layers				,							pt_model.transformer.resblocks						)






def        lowerCamelCase_  (				UpperCAmelCase_				:  int				,							UpperCAmelCase_				:  Tuple						)    ->					Union[str, Any]:




	'''simple docstring'''




	_UpperCamelCase :		Any								=    pt_model.text_projection.data.T

	# copy text encoder
	copy_encoder(hf_model.text_model				,							UpperCAmelCase_						)






def        lowerCamelCase_  (				UpperCAmelCase_				:  str				,							UpperCAmelCase_				:  Dict						)    ->					Optional[int]:




	'''simple docstring'''




	_UpperCamelCase :		str								=    pt_model.visual.proj.data.T

	# copy layer norms
	copy_linear(hf_model.vision_model.pre_layrnorm				,							pt_model.visual.ln_pre						)
	copy_linear(hf_model.vision_model.post_layernorm				,							pt_model.visual.ln_post						)

	# copy embeds
	_UpperCamelCase :		Tuple								=    pt_model.visual.conva.weight.data
	_UpperCamelCase :		Dict								=    pt_model.visual.class_embedding
	_UpperCamelCase :		str								=    pt_model.visual.positional_embedding.data

	# copy encoder
	copy_layers(hf_model.vision_model.encoder.layers				,							pt_model.visual.transformer.resblocks						)






@torch.no_grad()
def        lowerCamelCase_  (				UpperCAmelCase_				:  Dict				,							UpperCAmelCase_				:  str				,							UpperCAmelCase_				:  List[str]=None						)    ->					Union[str, Any]:




	'''simple docstring'''




	if config_path is not None:
		_UpperCamelCase :		Dict								=    CLIPConfig.from_pretrained(UpperCAmelCase_						)
	else:
		_UpperCamelCase :		Tuple								=    CLIPConfig(projection_dim=5_1_2				,							text_config={}				,							vision_config={}						)

	_UpperCamelCase :		Tuple								=    CLIPModel(UpperCAmelCase_						).eval()

	_UpperCamelCase						,		_UpperCamelCase :		Optional[Any]								=    load(UpperCAmelCase_				,							device='cpu'				,							jit=UpperCAmelCase_						)
	_UpperCamelCase :		Optional[Any]								=    pt_model.eval()

	copy_text_model_and_projection(UpperCAmelCase_				,							UpperCAmelCase_						)
	copy_vison_model_and_projection(UpperCAmelCase_				,							UpperCAmelCase_						)
	_UpperCamelCase :		Optional[Any]								=    pt_model.logit_scale

	_UpperCamelCase :		Optional[int]								=    torch.arange(0				,							7_7						).unsqueeze(0						)
	_UpperCamelCase :		Dict								=    torch.randn(1				,							3				,							2_2_4				,							2_2_4						)

	_UpperCamelCase :		Optional[Any]								=    hf_model(input_ids=UpperCAmelCase_				,							pixel_values=UpperCAmelCase_				,							return_dict=UpperCAmelCase_						)
	_UpperCamelCase :		Optional[Any]								=    hf_outputs.logits_per_image
	_UpperCamelCase :		Optional[int]								=    hf_outputs.logits_per_text
	_UpperCamelCase						,		_UpperCamelCase :		int								=    pt_model(UpperCAmelCase_				,							UpperCAmelCase_						)

	assert torch.allclose(UpperCAmelCase_				,							UpperCAmelCase_				,							atol=1e-3						)
	assert torch.allclose(UpperCAmelCase_				,							UpperCAmelCase_				,							atol=1e-3						)

	hf_model.save_pretrained(UpperCAmelCase_						)


if __name__ == "__main__":
							lowerCAmelCase__             =		argparse.ArgumentParser()
							parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
							parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to fairseq checkpoint""")
							parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
							lowerCAmelCase__             =		parser.parse_args()

							convert_clip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path)




