





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__UpperCamelCase      : Any					      =   logging.get_logger(__name__)

__UpperCamelCase      : Union[str, Any]					      =   {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__UpperCamelCase      : Union[str, Any]					      =   {
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
}

__UpperCamelCase      : Optional[Any]					      =   {
    "Salesforce/codegen-350M-mono": 2_048,
}





@lru_cache()
def 	__A	(			)   -> Optional[Any]:
	a		       =							(
	    list(range(ord("""!"""					)							,    ord("""~"""					) + 1					)					) + list(range(ord("""¡"""					)							,    ord("""¬"""					) + 1					)					) + list(range(ord("""®"""					)							,    ord("""ÿ"""					) + 1					)					)
	)
	a		       =							bs[:]
	a		       =							0
	for b in range(2**8					):
		if b not in bs:
			bs.append(lowercase__					)
			cs.append(2**8 + n					)
			n += 1
	a		       =							[chr(lowercase__					) for n in cs]
	return dict(zip(lowercase__							,    lowercase__					)					)





def 	__A	(			__lowerCamelCase					)   -> Dict:
	a		       =							set()
	a		       =							word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)					)
		a		       =							char
	return pairs



class 	__lowerCAmelCase			(			A_				):
							UpperCamelCase__        =						VOCAB_FILES_NAMES
							UpperCamelCase__        =						PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase__        =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase__        =						['''input_ids''', '''attention_mask''']
							def __init__(		self		:int	,   __magic_name__		:List[str]	,   __magic_name__		:Union[str, Any]	,   __magic_name__		:Optional[int]="replace"	,   __magic_name__		:Dict="<|endoftext|>"	,   __magic_name__		:int="<|endoftext|>"	,   __magic_name__		:Any="<|endoftext|>"	,   __magic_name__		:str=None	,   __magic_name__		:Optional[Any]=False	,   __magic_name__		:Dict=False	,   **__magic_name__		:Optional[Any]	,   ):







								'''simple docstring'''







								a		       =							AddedToken(__magic_name__	,   lstrip=__magic_name__	,   rstrip=__magic_name__       ) if isinstance(__magic_name__	,   __magic_name__       ) else bos_token
								a		       =							AddedToken(__magic_name__	,   lstrip=__magic_name__	,   rstrip=__magic_name__       ) if isinstance(__magic_name__	,   __magic_name__       ) else eos_token
								a		       =							AddedToken(__magic_name__	,   lstrip=__magic_name__	,   rstrip=__magic_name__       ) if isinstance(__magic_name__	,   __magic_name__       ) else unk_token
								a		       =							AddedToken(__magic_name__	,   lstrip=__magic_name__	,   rstrip=__magic_name__       ) if isinstance(__magic_name__	,   __magic_name__       ) else pad_token
								super().__init__(
								    errors=__magic_name__	,   unk_token=__magic_name__	,   bos_token=__magic_name__	,   eos_token=__magic_name__	,   pad_token=__magic_name__	,   add_prefix_space=__magic_name__	,   add_bos_token=__magic_name__	,   **__magic_name__	,   )
								a		       =							add_bos_token

								with open(__magic_name__	,   encoding="""utf-8"""       ) as vocab_handle:
									a		       =							json.load(__magic_name__       )
								a		       =							{v: k for k, v in self.encoder.items()}
								a		       =							errors  # how to handle errors in decoding
								a		       =							bytes_to_unicode()
								a		       =							{v: k for k, v in self.byte_encoder.items()}
								with open(__magic_name__	,   encoding="""utf-8"""       ) as merges_handle:
									a		       =							merges_handle.read().split("""\n"""       )[1:-1]
								a		       =							[tuple(merge.split()       ) for merge in bpe_merges]
								a		       =							dict(zip(__magic_name__	,   range(len(__magic_name__       )       )       )       )
								a		       =							{}
								a		       =							add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								a		       =							re.compile(r"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""       )
							@property
							def    lowerCamelCase__							(		self		:Dict       ):







								'''simple docstring'''







								return len(self.encoder       )
							def    lowerCamelCase__							(		self		:int       ):







								'''simple docstring'''







								return dict(self.encoder	,   **self.added_tokens_encoder       )
							def    lowerCamelCase__							(		self		:Any	,   __magic_name__		:Any       ):







								'''simple docstring'''







								if token in self.cache:
									return self.cache[token]
								a		       =							tuple(__magic_name__       )
								a		       =							get_pairs(__magic_name__       )

								if not pairs:
									return token

								while True:
									a		       =							min(__magic_name__	,   key=lambda __magic_name__       : self.bpe_ranks.get(__magic_name__	,   float("""inf"""       )       )       )
									if bigram not in self.bpe_ranks:
										break
									a       ,				a		       =							bigram
									a		       =							[]
									a		       =							0
									while i < len(__magic_name__       ):
										try:
											a		       =							word.index(__magic_name__	,   __magic_name__       )
										except ValueError:
											new_word.extend(word[i:]       )
											break
										else:
											new_word.extend(word[i:j]       )
											a		       =							j

										if word[i] == first and i < len(__magic_name__       ) - 1 and word[i + 1] == second:
											new_word.append(first + second       )
											i += 2
										else:
											new_word.append(word[i]       )
											i += 1
									a		       =							tuple(__magic_name__       )
									a		       =							new_word
									if len(__magic_name__       ) == 1:
										break
									else:
										a		       =							get_pairs(__magic_name__       )
								a		       =							""" """.join(__magic_name__       )
								a		       =							word
								return word
							def    lowerCamelCase__							(		self		:Any	,   __magic_name__		:int	,   __magic_name__		:List[Any]=None       ):







								'''simple docstring'''







								if self.add_bos_token:
									a		       =							[self.bos_token_id]
								else:
									a		       =							[]

								a		       =							bos_token_ids + token_ids_a

								if token_ids_a is None:
									return output

								return output + bos_token_ids + token_ids_a
							def    lowerCamelCase__							(		self		:str	,   __magic_name__		:Optional[int]       ):







								'''simple docstring'''







								a		       =							[]
								for token in re.findall(self.pat	,   __magic_name__       ):
									a		       =							"""""".join(
									    self.byte_encoder[b] for b in token.encode("""utf-8"""       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
									bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__magic_name__       ).split(""" """       )       )
								return bpe_tokens
							def    lowerCamelCase__							(		self		:List[Any]	,   __magic_name__		:List[Any]       ):







								'''simple docstring'''







								return self.encoder.get(__magic_name__	,   self.encoder.get(self.unk_token       )       )
							def    lowerCamelCase__							(		self		:Tuple	,   __magic_name__		:List[Any]       ):







								'''simple docstring'''







								return self.decoder.get(__magic_name__       )
							def    lowerCamelCase__							(		self		:Union[str, Any]	,   __magic_name__		:List[Any]       ):







								'''simple docstring'''







								a		       =							"""""".join(__magic_name__       )
								a		       =							bytearray([self.byte_decoder[c] for c in text]       ).decode("""utf-8"""	,   errors=self.errors       )
								return text
							def    lowerCamelCase__							(		self		:List[Any]	,   __magic_name__		:str	,   __magic_name__		:List[str] = None       ):







								'''simple docstring'''







								if not os.path.isdir(__magic_name__       ):
									logger.error(F'Vocabulary path ({save_directory}) should be a directory'       )
									return
								a		       =							os.path.join(
								    __magic_name__	,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
								a		       =							os.path.join(
								    __magic_name__	,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

								with open(__magic_name__	,   """w"""	,   encoding="""utf-8"""       ) as f:
									f.write(json.dumps(self.encoder	,   indent=2	,   sort_keys=__magic_name__	,   ensure_ascii=__magic_name__       ) + """\n"""       )

								a		       =							0
								with open(__magic_name__	,   """w"""	,   encoding="""utf-8"""       ) as writer:
									writer.write("""#version: 0.2\n"""       )
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,   key=lambda __magic_name__       : kv[1]       ):
										if index != token_index:
											logger.warning(
											    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
											    """ Please check that the tokenizer is not corrupted!"""       )
											a		       =							token_index
										writer.write(""" """.join(__magic_name__       ) + """\n"""       )
										index += 1

								return vocab_file, merge_file
							def    lowerCamelCase__							(		self		:Dict	,   __magic_name__		:Tuple	,   __magic_name__		:Union[str, Any]=False	,   **__magic_name__		:List[Any]       ):







								'''simple docstring'''







								a		       =							kwargs.pop("""add_prefix_space"""	,   self.add_prefix_space       )
								if is_split_into_words or add_prefix_space:
									a		       =							""" """ + text
								return (text, kwargs)
							def    lowerCamelCase__							(		self		:Optional[int]	,   __magic_name__		:Union[str, Any]	,   __magic_name__		:List[str] = False	,   __magic_name__		:Optional[int] = None	,   __magic_name__		:List[Any] = None	,   **__magic_name__		:Union[str, Any]	,   ):







								'''simple docstring'''







								a		       =							super()._decode(
								    token_ids=__magic_name__	,   skip_special_tokens=__magic_name__	,   clean_up_tokenization_spaces=__magic_name__	,   **__magic_name__	,   )

								if truncate_before_pattern is not None and len(__magic_name__       ) > 0:
									a		       =							self.truncate(__magic_name__	,   __magic_name__       )

								return decoded_text





							def    lowerCamelCase__							(		self		:Optional[int]	,   __magic_name__		:List[Any]	,   __magic_name__		:int       ):







								'''simple docstring'''







								def find_re(__magic_name__		:Optional[Any]	,   __magic_name__		:List[Any]	,   __magic_name__		:int       ):
									a		       =							pattern.search(__magic_name__	,   __magic_name__       )
									return m.start() if m else -1

								a		       =							[re.compile(__magic_name__	,   re.MULTILINE       ) for pattern in truncate_before_pattern]

								a		       =							list(re.finditer("""^print"""	,   __magic_name__	,   re.MULTILINE       )       )

								if len(__magic_name__       ) > 1:
									a		       =							completion[: prints[1].start()]

								a		       =							list(re.finditer("""^def"""	,   __magic_name__	,   re.MULTILINE       )       )

								if len(__magic_name__       ) > 1:
									a		       =							completion[: defs[1].start()]

								a		       =							0

								a		       =							[
								    pos for pos in [find_re(__magic_name__	,   __magic_name__	,   __magic_name__       ) for terminal in terminals] if pos != -1
								]

								if len(__magic_name__       ) > 0:
									return completion[: min(__magic_name__       )]
								else:
									return completion







import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ConvNextConfig, ConvNextForImageClassification, ConvNextImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
__UpperCamelCase      : List[Any]					      =   logging.get_logger(__name__)





def 	__A	(			__lowerCamelCase					)   -> Optional[int]:
	a		       =							ConvNextConfig()

	if "tiny" in checkpoint_url:
		a		       =							[3, 3, 9, 3]
		a		       =							[96, 192, 384, 768]
	if "small" in checkpoint_url:
		a		       =							[3, 3, 27, 3]
		a		       =							[96, 192, 384, 768]
	if "base" in checkpoint_url:
		a		       =							[3, 3, 27, 3]
		a		       =							[128, 256, 512, 1024]
	if "large" in checkpoint_url:
		a		       =							[3, 3, 27, 3]
		a		       =							[192, 384, 768, 1536]
	if "xlarge" in checkpoint_url:
		a		       =							[3, 3, 27, 3]
		a		       =							[256, 512, 1024, 2048]

	if "1k" in checkpoint_url:
		a		       =							1000
		a		       =							"""imagenet-1k-id2label.json"""
		a		       =							(1, 1000)
	else:
		a		       =							2_1841
		a		       =							"""imagenet-22k-id2label.json"""
		a		       =							(1, 2_1841)

	a		       =							"""huggingface/label-files"""
	a		       =							num_labels
	a		       =							json.load(open(hf_hub_download(__lowerCamelCase							,    __lowerCamelCase							,    repo_type="""dataset"""					)							,    """r"""					)					)
	a		       =							{int(__lowerCamelCase					): v for k, v in idalabel.items()}
	if "1k" not in checkpoint_url:
		# this dataset contains 21843 labels but the model only has 21841
		# we delete the classes as mentioned in https://github.com/google-research/big_transfer/issues/18
		del idalabel[9205]
		del idalabel[1_5027]
	a		       =							idalabel
	a		       =							{v: k for k, v in idalabel.items()}
	a		       =							hidden_sizes
	a		       =							depths

	return config, expected_shape





def 	__A	(			__lowerCamelCase					)   -> int:
	if "downsample_layers.0.0" in name:
		a		       =							name.replace("""downsample_layers.0.0"""							,    """embeddings.patch_embeddings"""					)
	if "downsample_layers.0.1" in name:
		a		       =							name.replace("""downsample_layers.0.1"""							,    """embeddings.norm"""					)  # we rename to layernorm later on
	if "downsample_layers.1.0" in name:
		a		       =							name.replace("""downsample_layers.1.0"""							,    """stages.1.downsampling_layer.0"""					)
	if "downsample_layers.1.1" in name:
		a		       =							name.replace("""downsample_layers.1.1"""							,    """stages.1.downsampling_layer.1"""					)
	if "downsample_layers.2.0" in name:
		a		       =							name.replace("""downsample_layers.2.0"""							,    """stages.2.downsampling_layer.0"""					)
	if "downsample_layers.2.1" in name:
		a		       =							name.replace("""downsample_layers.2.1"""							,    """stages.2.downsampling_layer.1"""					)
	if "downsample_layers.3.0" in name:
		a		       =							name.replace("""downsample_layers.3.0"""							,    """stages.3.downsampling_layer.0"""					)
	if "downsample_layers.3.1" in name:
		a		       =							name.replace("""downsample_layers.3.1"""							,    """stages.3.downsampling_layer.1"""					)
	if "stages" in name and "downsampling_layer" not in name:
		# stages.0.0. for instance should be renamed to stages.0.layers.0.
		a		       =							name[: len("""stages.0"""					)] + """.layers""" + name[len("""stages.0"""					) :]
	if "stages" in name:
		a		       =							name.replace("""stages"""							,    """encoder.stages"""					)
	if "norm" in name:
		a		       =							name.replace("""norm"""							,    """layernorm"""					)
	if "gamma" in name:
		a		       =							name.replace("""gamma"""							,    """layer_scale_parameter"""					)
	if "head" in name:
		a		       =							name.replace("""head"""							,    """classifier"""					)

	return name





def 	__A	(			)   -> Optional[Any]:
	a		       =							"""http://images.cocodataset.org/val2017/000000039769.jpg"""
	a		       =							Image.open(requests.get(__lowerCamelCase							,    stream=__lowerCamelCase					).raw					)
	return im





@torch.no_grad()
def 	__A	(			__lowerCamelCase							,    __lowerCamelCase					)   -> List[Any]:
	a       ,				a		       =							get_convnext_config(__lowerCamelCase					)
	# load original state_dict from URL
	a		       =							torch.hub.load_state_dict_from_url(__lowerCamelCase					)["""model"""]
	# rename keys
	for key in state_dict.copy().keys():
		a		       =							state_dict.pop(__lowerCamelCase					)
		a		       =							val
	# add prefix to all keys expect classifier head
	for key in state_dict.copy().keys():
		a		       =							state_dict.pop(__lowerCamelCase					)
		if not key.startswith("""classifier"""					):
			a		       =							"""convnext.""" + key
		a		       =							val

	# load HuggingFace model
	a		       =							ConvNextForImageClassification(__lowerCamelCase					)
	model.load_state_dict(__lowerCamelCase					)
	model.eval()

	# Check outputs on an image, prepared by ConvNextImageProcessor
	a		       =							224 if """224""" in checkpoint_url else 384
	a		       =							ConvNextImageProcessor(size=__lowerCamelCase					)
	a		       =							image_processor(images=prepare_img()							,    return_tensors="""pt"""					).pixel_values

	a		       =							model(__lowerCamelCase					).logits

	# note: the logits below were obtained without center cropping
	if checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pth":
		a		       =							torch.tensor([-0.1210, -0.6605, 0.1918]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_small_1k_224_ema.pth":
		a		       =							torch.tensor([-0.4473, -0.1847, -0.6365]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_1k_224_ema.pth":
		a		       =							torch.tensor([0.4525, 0.7539, 0.0308]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_1k_384.pth":
		a		       =							torch.tensor([0.3561, 0.6350, -0.0384]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_1k_224_ema.pth":
		a		       =							torch.tensor([0.4174, -0.0989, 0.1489]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_1k_384.pth":
		a		       =							torch.tensor([0.2513, -0.1349, -0.1613]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_224.pth":
		a		       =							torch.tensor([1.2980, 0.3631, -0.1198]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_224.pth":
		a		       =							torch.tensor([1.2963, 0.1227, 0.1723]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_224.pth":
		a		       =							torch.tensor([1.7956, 0.8390, 0.2820]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_1k_224.pth":
		a		       =							torch.tensor([-0.2822, -0.0502, -0.0878]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_1k_384.pth":
		a		       =							torch.tensor([-0.5672, -0.0730, -0.4348]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_1k_224.pth":
		a		       =							torch.tensor([0.2681, 0.2365, 0.6246]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_1k_384.pth":
		a		       =							torch.tensor([-0.2642, 0.3931, 0.5116]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_1k_224_ema.pth":
		a		       =							torch.tensor([-0.6677, -0.1873, -0.8379]					)
	elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_1k_384_ema.pth":
		a		       =							torch.tensor([-0.7749, -0.2967, -0.6444]					)
	else:
		raise ValueError(f'Unknown URL: {checkpoint_url}'					)

	assert torch.allclose(logits[0, :3]							,    __lowerCamelCase							,    atol=1E-3					)
	assert logits.shape == expected_shape

	Path(__lowerCamelCase					).mkdir(exist_ok=__lowerCamelCase					)
	print(f'Saving model to {pytorch_dump_folder_path}'					)
	model.save_pretrained(__lowerCamelCase					)
	print(f'Saving image processor to {pytorch_dump_folder_path}'					)
	image_processor.save_pretrained(__lowerCamelCase					)

	print("""Pushing model to the hub..."""					)
	a		       =							"""convnext"""
	if "tiny" in checkpoint_url:
		model_name += "-tiny"
	elif "small" in checkpoint_url:
		model_name += "-small"
	elif "base" in checkpoint_url:
		model_name += "-base"
	elif "xlarge" in checkpoint_url:
		model_name += "-xlarge"
	elif "large" in checkpoint_url:
		model_name += "-large"
	if "224" in checkpoint_url:
		model_name += "-224"
	elif "384" in checkpoint_url:
		model_name += "-384"
	if "22k" in checkpoint_url and "1k" not in checkpoint_url:
		model_name += "-22k"
	if "22k" in checkpoint_url and "1k" in checkpoint_url:
		model_name += "-22k-1k"

	model.push_to_hub(
	    repo_path_or_name=Path(__lowerCamelCase							,    __lowerCamelCase					)							,    organization="""nielsr"""							,    commit_message="""Add model"""							,    )


if __name__ == "__main__":
		__UpperCamelCase      : int					      =   argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    "--checkpoint_url",
		    default="https://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pth",
		    type=str,
		    help="URL of the original ConvNeXT checkpoint you'd like to convert.",
		)
		parser.add_argument(
		    "--pytorch_dump_folder_path",
		    default=None,
		    type=str,
		    required=True,
		    help="Path to the output PyTorch model directory.",
		)

		__UpperCamelCase      : List[Any]					      =   parser.parse_args()
		convert_convnext_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)

