




"""simple docstring"""





import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


_lowercase						   =     logging.get_logger(__name__)

_lowercase						   =     {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase						   =     {
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

_lowercase						   =     {
    """microsoft/deberta-base""": 5_12,
    """microsoft/deberta-large""": 5_12,
    """microsoft/deberta-xlarge""": 5_12,
    """microsoft/deberta-base-mnli""": 5_12,
    """microsoft/deberta-large-mnli""": 5_12,
    """microsoft/deberta-xlarge-mnli""": 5_12,
}

_lowercase						   =     {
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}

class    __lowercase							(      _snake_case						):


				'''simple docstring'''


				_lowerCAmelCase						      =VOCAB_FILES_NAMES
				_lowerCAmelCase						      =PRETRAINED_VOCAB_FILES_MAP
				_lowerCAmelCase						      =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowerCAmelCase						      =["""input_ids""", """attention_mask""", """token_type_ids"""]
				_lowerCAmelCase						      =DebertaTokenizer

				def __init__( self :  List[str]   ,SCREAMING_SNAKE_CASE_ :  Optional[Any]=None   ,SCREAMING_SNAKE_CASE_ :  Optional[Any]=None   ,SCREAMING_SNAKE_CASE_ :  Optional[Any]=None   ,SCREAMING_SNAKE_CASE_ :  Dict="replace"   ,SCREAMING_SNAKE_CASE_ :  Dict="[CLS]"   ,SCREAMING_SNAKE_CASE_ :  Union[str, Any]="[SEP]"   ,SCREAMING_SNAKE_CASE_ :  Any="[SEP]"   ,SCREAMING_SNAKE_CASE_ :  List[Any]="[CLS]"   ,SCREAMING_SNAKE_CASE_ :  str="[UNK]"   ,SCREAMING_SNAKE_CASE_ :  Tuple="[PAD]"   ,SCREAMING_SNAKE_CASE_ :  Union[str, Any]="[MASK]"   ,SCREAMING_SNAKE_CASE_ :  List[Any]=False   ,**SCREAMING_SNAKE_CASE_ :  Any   ,):
										super().__init__(
										    lowercase_   ,lowercase_   ,tokenizer_file=lowercase_   ,errors=lowercase_   ,bos_token=lowercase_   ,eos_token=lowercase_   ,unk_token=lowercase_   ,sep_token=lowercase_   ,cls_token=lowercase_   ,pad_token=lowercase_   ,mask_token=lowercase_   ,add_prefix_space=lowercase_   ,**lowercase_   ,)
										snake_case__							:       Union[str, Any]				      =       kwargs.pop('''add_bos_token'''   ,lowercase_   )

										snake_case__							:       List[str]				      =       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
										if pre_tok_state.get('''add_prefix_space'''   ,lowercase_   ) != add_prefix_space:
																snake_case__							:       Tuple				      =       getattr(lowercase_   ,pre_tok_state.pop('''type'''   )   )
																snake_case__							:       str				      =       add_prefix_space
																snake_case__							:       Dict				      =       pre_tok_class(**lowercase_   )

										snake_case__							:       List[Any]				      =       add_prefix_space

				@property
				def    _lowerCamelCase       ( self :  Dict   ):
										if self._mask_token is None:
																if self.verbose:
																						logger.error('''Using mask_token, but it is not set yet.'''   )
																return None
										return str(self._mask_token   )

				@mask_token.setter
				def    _lowerCamelCase       ( self :  Dict   ,SCREAMING_SNAKE_CASE_ :  Union[str, Any]   ):
										snake_case__							:       Union[str, Any]				      =       AddedToken(lowercase_   ,lstrip=lowercase_   ,rstrip=lowercase_   ) if isinstance(lowercase_   ,lowercase_   ) else value
										snake_case__							:       Union[str, Any]				      =       value

				def    _lowerCamelCase       ( self :  Optional[int]   ,SCREAMING_SNAKE_CASE_ :  List[int]   ,SCREAMING_SNAKE_CASE_ :  Optional[List[int]] = None   ):
										if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										snake_case__							:       Union[str, Any]				      =       [self.cls_token_id]
										snake_case__							:       List[str]				      =       [self.sep_token_id]
										return cls + token_ids_a + sep + token_ids_a + sep

				def    _lowerCamelCase       ( self :  Optional[int]   ,SCREAMING_SNAKE_CASE_ :  List[int]   ,SCREAMING_SNAKE_CASE_ :  Optional[List[int]] = None   ):
										snake_case__							:       List[Any]				      =       [self.sep_token_id]
										snake_case__							:       Optional[int]				      =       [self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep   ) * [0]
										return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]

				def    _lowerCamelCase       ( self :  Optional[Any]   ,*SCREAMING_SNAKE_CASE_ :  Optional[int]   ,**SCREAMING_SNAKE_CASE_ :  Dict   ):
										snake_case__							:       List[str]				      =       kwargs.get('''is_split_into_words'''   ,lowercase_   )
										assert self.add_prefix_space or not is_split_into_words, (
										    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*lowercase_   ,**lowercase_   )

				def    _lowerCamelCase       ( self :  Any   ,*SCREAMING_SNAKE_CASE_ :  Union[str, Any]   ,**SCREAMING_SNAKE_CASE_ :  Tuple   ):
										snake_case__							:       List[Any]				      =       kwargs.get('''is_split_into_words'''   ,lowercase_   )

										assert self.add_prefix_space or not is_split_into_words, (
										    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*lowercase_   ,**lowercase_   )

				def    _lowerCamelCase       ( self :  Dict   ,SCREAMING_SNAKE_CASE_ :  str   ,SCREAMING_SNAKE_CASE_ :  Optional[str] = None   ):
										snake_case__							:       Dict				      =       self._tokenizer.model.save(lowercase_   ,name=lowercase_   )
										return tuple(lowercase_   )





				def    _lowerCamelCase       ( self :  str   ,SCREAMING_SNAKE_CASE_ :  "Conversation"   ):
										snake_case__							:       Optional[int]				      =       []
										for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(lowercase_   ,add_special_tokens=lowercase_   ) + [self.eos_token_id]   )

										if len(lowercase_   ) > self.model_max_length:
																snake_case__							:       int				      =       input_ids[-self.model_max_length :]
										return input_ids





'''simple docstring'''

import argparse

import torch

from transformers import ChineseCLIPConfig, ChineseCLIPModel




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_    ,			UpperCAmelCase_      ):
 UpperCAmelCase   ,					UpperCAmelCase   ,					UpperCAmelCase			:      Optional[Any]		   =					pt_weights[F"""{prefix}.in_proj_weight"""].chunk(3    ,			dim=0      )
 UpperCAmelCase   ,					UpperCAmelCase   ,					UpperCAmelCase			:      str		   =					pt_weights[F"""{prefix}.in_proj_bias"""].chunk(3    ,			dim=0      )

 UpperCAmelCase			:      Dict		   =					pt_weights[F"""{prefix}.out_proj.weight"""]
 UpperCAmelCase			:      Tuple		   =					pt_weights[F"""{prefix}.out_proj.bias"""]

 UpperCAmelCase			:      Dict		   =					q_proj
 UpperCAmelCase			:      Optional[Any]		   =					q_proj_bias

 UpperCAmelCase			:      Union[str, Any]		   =					k_proj
 UpperCAmelCase			:      Optional[Any]		   =					k_proj_bias

 UpperCAmelCase			:      Dict		   =					v_proj
 UpperCAmelCase			:      Tuple		   =					v_proj_bias

 UpperCAmelCase			:      List[Any]		   =					out_proj_weights
 UpperCAmelCase			:      Any		   =					out_proj_bias




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_    ,			UpperCAmelCase_      ):
 copy_linear(hf_mlp.fca    ,			UpperCAmelCase_    ,			F"""{prefix}.c_fc"""      )
 copy_linear(hf_mlp.fca    ,			UpperCAmelCase_    ,			F"""{prefix}.c_proj"""      )




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_    ,			UpperCAmelCase_      ):
 UpperCAmelCase			:      Tuple		   =					pt_weights[F"""{prefix}.weight"""].data
 UpperCAmelCase			:      Dict		   =					pt_weights[F"""{prefix}.bias"""].data




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_    ,			UpperCAmelCase_      ):
 # copy layer norms
 copy_linear(hf_layer.layer_norma    ,			UpperCAmelCase_    ,			F"""{prefix}.ln_1"""      )
 copy_linear(hf_layer.layer_norma    ,			UpperCAmelCase_    ,			F"""{prefix}.ln_2"""      )

 # copy MLP
 copy_mlp(hf_layer.mlp    ,			UpperCAmelCase_    ,			F"""{prefix}.mlp"""      )

 # copy attn
 copy_attn_layer(hf_layer.self_attn    ,			UpperCAmelCase_    ,			F"""{prefix}.attn"""      )




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_    ,			UpperCAmelCase_      ):
 for layer_id, hf_layer in enumerate(UpperCAmelCase_      ):
  copy_layer(UpperCAmelCase_    ,			UpperCAmelCase_    ,			F"""{prefix}.{layer_id}"""      )




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_      ):
 # copy projection
 UpperCAmelCase			:      Optional[int]		   =					pt_weights['text_projection'].data.T

 # copy text encoder
 for name, param in hf_model.text_model.named_parameters():
  UpperCAmelCase			:      int		   =					pt_weights[F"""bert.{name}"""].data




def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_      ):
 # copy projection
 UpperCAmelCase			:      Optional[int]		   =					pt_weights['visual.proj'].data.T

 # copy layer norms
 copy_linear(hf_model.vision_model.pre_layrnorm    ,			UpperCAmelCase_    ,			'visual.ln_pre'      )
 copy_linear(hf_model.vision_model.post_layernorm    ,			UpperCAmelCase_    ,			'visual.ln_post'      )

 # copy embeddings
 UpperCAmelCase			:      List[str]		   =					pt_weights['visual.conv1.weight'].data
 UpperCAmelCase			:      Tuple		   =					pt_weights['visual.class_embedding'].data
 UpperCAmelCase			:      str		   =					pt_weights['visual.positional_embedding'].data

 # copy encoder
 copy_layers(hf_model.vision_model.encoder.layers    ,			UpperCAmelCase_    ,			'visual.transformer.resblocks'      )




@torch.no_grad()
def  UpperCamelCase(	UpperCAmelCase_    ,			UpperCAmelCase_    ,			UpperCAmelCase_=None      ):
 assert config_path is not None, "Please specify the ChineseCLIP model config of the corresponding model size."
 UpperCAmelCase			:      str		   =					ChineseCLIPConfig.from_pretrained(UpperCAmelCase_      )

 UpperCAmelCase			:      List[Any]		   =					ChineseCLIPModel(UpperCAmelCase_      ).eval()

 UpperCAmelCase			:      Optional[Any]		   =					torch.load(UpperCAmelCase_    ,			map_location='cpu'      )['state_dict']
 UpperCAmelCase			:      Any		   =					{(name[7:] if name.startswith('module.'      ) else name): value for name, value in pt_weights.items()}

 copy_text_model_and_projection(UpperCAmelCase_    ,			UpperCAmelCase_      )
 copy_vision_model_and_projection(UpperCAmelCase_    ,			UpperCAmelCase_      )
 UpperCAmelCase			:      Optional[int]		   =					pt_weights['logit_scale'].data

 hf_model.save_pretrained(UpperCAmelCase_      )


if __name__ == "__main__":
 lowercase__            =     argparse.ArgumentParser()
 parser.add_argument(
     "--pytorch_dump_folder_path",
     default=None,
     type=str,
     help="Path to the output folder storing converted hf PyTorch model.",
 )
 parser.add_argument(
     "--checkpoint_path", default=None, type=str, help="Path to original github format ChineseCLIP checkpoint."
 )
 parser.add_argument(
     "--config_path", default=None, required=True, type=str, help="Path to hf config.json of model to convert."
 )
 lowercase__            =     parser.parse_args()

 convert_chinese_clip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path)
 print("The conversion is finished!")




