




'''simple docstring'''






import argparse

import torch
from clip import load

from transformers import CLIPConfig, CLIPModel




def 		a_(			lowerCamelCase ,					lowerCamelCase   )					->List[Any]:






       '''simple docstring'''




       lowercase,  lowercase,  lowercase	:		Optional[int]    =   pt_attn_layer.in_proj_weight.chunk(3 ,					dim=0   )
       lowercase,  lowercase,  lowercase	:		Dict    =   pt_attn_layer.in_proj_bias.chunk(3 ,					dim=0   )

       lowercase	:		Optional[Any]    =   pt_attn_layer.out_proj.weight
       lowercase	:		Tuple    =   pt_attn_layer.out_proj.bias

       lowercase	:		Dict    =   q_proj
       lowercase	:		List[Any]    =   q_proj_bias

       lowercase	:		Union[str, Any]    =   k_proj
       lowercase	:		Any    =   k_proj_bias

       lowercase	:		Dict    =   v_proj
       lowercase	:		int    =   v_proj_bias

       lowercase	:		Union[str, Any]    =   out_proj_weights
       lowercase	:		Union[str, Any]    =   out_proj_bias




def 		a_(			lowerCamelCase ,					lowerCamelCase   )					->Union[str, Any]:






       '''simple docstring'''




       copy_linear(hf_mlp.fca ,					pt_mlp.c_fc   )
       copy_linear(hf_mlp.fca ,					pt_mlp.c_proj   )




def 		a_(			lowerCamelCase ,					lowerCamelCase   )					->List[str]:






       '''simple docstring'''




       lowercase	:		Optional[int]    =   pt_linear.weight
       lowercase	:		Tuple    =   pt_linear.bias




def 		a_(			lowerCamelCase ,					lowerCamelCase   )					->Union[str, Any]:






       '''simple docstring'''




       copy_linear(hf_layer.layer_norma ,					pt_layer.ln_a   )
       copy_linear(hf_layer.layer_norma ,					pt_layer.ln_a   )

       # copy MLP
       copy_mlp(hf_layer.mlp ,					pt_layer.mlp   )

       # copy attn
       copy_attn_layer(hf_layer.self_attn ,					pt_layer.attn   )




def 		a_(			lowerCamelCase ,					lowerCamelCase   )					->Union[str, Any]:






       '''simple docstring'''




       for hf_layer, pt_layer in zip(lowerCamelCase ,					lowerCamelCase   ):
              copy_layer(lowerCamelCase ,					lowerCamelCase   )




def 		a_(			lowerCamelCase ,					lowerCamelCase   )					->Optional[int]:






       '''simple docstring'''




       lowercase	:		int    =   pt_model.token_embedding.weight
       lowercase	:		List[str]    =   pt_model.positional_embedding

       # copy layer norm
       copy_linear(hf_encoder.final_layer_norm ,					pt_model.ln_final   )

       # copy hidden layers
       copy_layers(hf_encoder.encoder.layers ,					pt_model.transformer.resblocks   )




def 		a_(			lowerCamelCase ,					lowerCamelCase   )					->List[str]:






       '''simple docstring'''




       lowercase	:		Dict    =   pt_model.text_projection.data.T

       # copy text encoder
       copy_encoder(hf_model.text_model ,					lowerCamelCase   )




def 		a_(			lowerCamelCase ,					lowerCamelCase   )					->Dict:






       '''simple docstring'''




       lowercase	:		Optional[Any]    =   pt_model.visual.proj.data.T

       # copy layer norms
       copy_linear(hf_model.vision_model.pre_layrnorm ,					pt_model.visual.ln_pre   )
       copy_linear(hf_model.vision_model.post_layernorm ,					pt_model.visual.ln_post   )

       # copy embeds
       lowercase	:		int    =   pt_model.visual.conva.weight.data
       lowercase	:		List[str]    =   pt_model.visual.class_embedding
       lowercase	:		Union[str, Any]    =   pt_model.visual.positional_embedding.data

       # copy encoder
       copy_layers(hf_model.vision_model.encoder.layers ,					pt_model.visual.transformer.resblocks   )




@torch.no_grad()
def 		a_(			lowerCamelCase ,					lowerCamelCase ,					lowerCamelCase=None   )					->str:






       '''simple docstring'''




       if config_path is not None:
              lowercase	:		Tuple    =   CLIPConfig.from_pretrained(lowerCamelCase   )
       else:
              lowercase	:		Dict    =   CLIPConfig(projection_dim=512 ,					text_config={} ,					vision_config={}   )

       lowercase	:		int    =   CLIPModel(lowerCamelCase   ).eval()

       lowercase,  lowercase	:		int    =   load(lowerCamelCase ,					device='''cpu''' ,					jit=lowerCamelCase   )
       lowercase	:		Any    =   pt_model.eval()

       copy_text_model_and_projection(lowerCamelCase ,					lowerCamelCase   )
       copy_vison_model_and_projection(lowerCamelCase ,					lowerCamelCase   )
       lowercase	:		Any    =   pt_model.logit_scale

       lowercase	:		int    =   torch.arange(0 ,					77   ).unsqueeze(0   )
       lowercase	:		Tuple    =   torch.randn(1 ,					3 ,					224 ,					224   )

       lowercase	:		str    =   hf_model(input_ids=lowerCamelCase ,					pixel_values=lowerCamelCase ,					return_dict=lowerCamelCase   )
       lowercase	:		List[Any]    =   hf_outputs.logits_per_image
       lowercase	:		Optional[Any]    =   hf_outputs.logits_per_text
       lowercase,  lowercase	:		Dict    =   pt_model(lowerCamelCase ,					lowerCamelCase   )

       assert torch.allclose(lowerCamelCase ,					lowerCamelCase ,					atol=1E-3   )
       assert torch.allclose(lowerCamelCase ,					lowerCamelCase ,					atol=1E-3   )

       hf_model.save_pretrained(lowerCamelCase   )


if __name__ == "__main__":
      lowercase						:   Optional[Any]      						=							argparse.ArgumentParser()
      parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
      parser.add_argument('''--checkpoint_path''', default=None, type=str, help='''Path to fairseq checkpoint''')
      parser.add_argument('''--config_path''', default=None, type=str, help='''Path to hf config.json of model to convert''')
      lowercase						:   Any      						=							parser.parse_args()

      convert_clip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path)










'''simple docstring'''






from manim import *

class 	SCREAMING_SNAKE_CASE_		(							lowerCamelCase_					):



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		str    =   Rectangle(height=0.5 ,		width=0.5			)
        lowercase	:		Optional[Any]    =   Rectangle(height=0.4_6 ,		width=0.4_6			).set_stroke(width=0			)
        lowercase	:		str    =   Rectangle(height=0.2_5 ,		width=0.2_5			)

        lowercase	:		Any    =   [mem.copy() for i in range(6			)]
        lowercase	:		List[str]    =   [mem.copy() for i in range(6			)]
        lowercase	:		List[Any]    =   VGroup(*A			).arrange(A ,		buff=0			)
        lowercase	:		Any    =   VGroup(*A			).arrange(A ,		buff=0			)
        lowercase	:		str    =   VGroup(A ,		A			).arrange(A ,		buff=0			)
        lowercase	:		int    =   Text('''CPU''' ,		font_size=24			)
        lowercase	:		Dict    =   Group(A ,		A			).arrange(A ,		buff=0.5 ,		aligned_edge=A			)
        cpu.move_to([-2.5, -0.5, 0]			)
        self.add(A			)

        lowercase	:		List[str]    =   [mem.copy() for i in range(4			)]
        lowercase	:		Optional[Any]    =   VGroup(*A			).arrange(A ,		buff=0			)
        lowercase	:		Optional[int]    =   Text('''GPU''' ,		font_size=24			)
        lowercase	:		Optional[int]    =   Group(A ,		A			).arrange(A ,		buff=0.5 ,		aligned_edge=A			)
        gpu.move_to([-1, -1, 0]			)
        self.add(A			)

        lowercase	:		str    =   [mem.copy() for i in range(6			)]
        lowercase	:		int    =   VGroup(*A			).arrange(A ,		buff=0			)

        lowercase	:		int    =   Text('''Model''' ,		font_size=24			)
        lowercase	:		Tuple    =   Group(A ,		A			).arrange(A ,		buff=0.5 ,		aligned_edge=A			)
        model.move_to([3, -1.0, 0]			)
        self.add(A			)

        lowercase	:		str    =   []
        lowercase	:		int    =   []

        for i, rect in enumerate(A			):
               rect.set_stroke(A			)

               lowercase	:		List[Any]    =   Rectangle(height=0.4_6 / 4 ,		width=0.4_6 / 3			).set_stroke(width=0.0			).set_fill(A ,		opacity=0.7			)

               if i == 0:
                      cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT			) ,		buff=0.0_2 ,		direction=A			)
                      cpu_target.set_x(cpu_target.get_x() + 0.1			)
               elif i == 3:
                      cpu_target.next_to(model_cpu_arr[0] ,		direction=A ,		buff=0.0			)
               else:
                      cpu_target.next_to(model_cpu_arr[i - 1] ,		direction=A ,		buff=0.0			)
               self.add(A			)
               model_cpu_arr.append(A			)

        self.add(*A ,		*A			)

        lowercase	:		Optional[int]    =   [meta_mem.copy() for i in range(6			)]
        lowercase	:		List[str]    =   [meta_mem.copy() for i in range(6			)]
        lowercase	:		int    =   VGroup(*A			).arrange(A ,		buff=0			)
        lowercase	:		int    =   VGroup(*A			).arrange(A ,		buff=0			)
        lowercase	:		Any    =   VGroup(A ,		A			).arrange(A ,		buff=0			)
        lowercase	:		Union[str, Any]    =   Text('''Disk''' ,		font_size=24			)
        lowercase	:		Dict    =   Group(A ,		A			).arrange(A ,		buff=0.5 ,		aligned_edge=A			)
        disk.move_to([-4.0, -1.2_5, 0]			)
        self.add(A ,		A			)

        lowercase	:		List[str]    =   []

        for i in range(6			):
               lowercase	:		Tuple    =   fill.copy().set_fill(A ,		opacity=0.8			)
               target.move_to(disk_left_col_base[i]			).scale(0.5			)
               cpu_disk_arr.append(A			)

        self.add(*A			)

        lowercase	:		Any    =   Square(side_length=2.2			)
        key.move_to([-5, 2, 0]			)

        lowercase	:		Tuple    =   MarkupText(
            f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model""" ,		font_size=18 ,		)

        key_text.move_to([-5, 2.4, 0]			)

        self.add(A ,		A			)

        lowercase	:		List[Any]    =   MarkupText(
            f"""<span fgcolor='{BLUE}'>●</span> Checkpoint""" ,		font_size=18 ,		)

        blue_text.next_to(A ,		DOWN * 2.4 ,		aligned_edge=key_text.get_left()			)
        self.add(A			)

        lowercase	:		Optional[Any]    =   MarkupText(f"""The offloaded weights are all sent to the CPU.""" ,		font_size=24			)
        step_a.move_to([2, 2, 0]			)

        self.play(Write(A ,		run_time=3			)			)

        for i in range(6			):
               lowercase	:		List[str]    =   cpu_disk_arr[i]
               lowercase	:		Optional[Any]    =   rect.copy().set_fill(A ,		opacity=0.8			).scale(2.0			)
               cpa.generate_target()
               cpa.target.move_to(model_base[i]			)

               if i == 0:
                      rect.set_fill(A ,		opacity=0.8			)
                      rect.generate_target()
                      rect.target.move_to(cpu_left_col_base[0]			).scale(2.0			)

                      self.remove(
                          *A ,		*A ,		)

               else:
                      rect.generate_target()
                      rect.target.move_to(cpu_left_col_base[i]			).scale(2.0			)
               self.play(MoveToTarget(A			) ,		MoveToTarget(A			) ,		model_base[i].animate.set_stroke(A			)			)
        self.play(FadeOut(A			)			)

        lowercase	:		Optional[Any]    =   MarkupText(
            f"""Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed.""" ,		font_size=24 ,		)
        step_a.move_to([2, 2, 0]			)

        self.play(Write(A ,		run_time=3			)			)

        lowercase	:		Tuple    =   []
        lowercase	:		List[Any]    =   []
        for i in range(6			):
               lowercase	:		List[str]    =   Arrow(start=A ,		end=A ,		color=A ,		buff=0.5			)
               a.next_to(model_base[i].get_left() ,		A ,		buff=0.2			)
               arrows.append(A			)
               animations.append(Write(A			)			)
        self.play(*A			)
        self.wait()




