






"""simple docstring"""


import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata




def 				a (     UpperCamelCase_							:				str		)  ->		str:
 __A	:       Dict       =	min(re.search('^ +'     ,	UpperCamelCase_		).end() if t.startswith(' '		) else 0 for t in string.splitlines()		)
 return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(UpperCamelCase_		)]		)


_UpperCamelCase   :						Dict     						= """\
---
language:
- zh
- en
task_ids:
- sentiment-classification
---
# Begin of markdown

Some cool dataset card
"""

_UpperCamelCase   :						str     						= """\
---
---
# Begin of markdown

Some cool dataset card
"""


_UpperCamelCase   :						int     						= """\
# Begin of markdown

Some cool dataset card
"""






class __a   (   unittest.TestCase				):



  def 						A_				(self  :		Any   )		->							int:
   with tempfile.TemporaryDirectory() as tmp_dir:
    __A	:       str       =	Path(lowercase_   ) / 'README.md'
    with open(lowercase_				,			'w+'   ) as readme_file:
     readme_file.write(lowercase_   )
    __A	:       List[Any]       =	DatasetMetadata.from_readme(lowercase_   )
    self.assertDictEqual(lowercase_				,			{'language': ['zh', 'en'], 'task_ids': ['sentiment-classification']}   )

    with open(lowercase_				,			'w+'   ) as readme_file:
     readme_file.write(lowercase_   )
    __A	:       Union[str, Any]       =	DatasetMetadata.from_readme(lowercase_   )
    self.assertDictEqual(lowercase_				,			{}   )

    with open(lowercase_				,			'w+'   ) as readme_file:
     readme_file.write(lowercase_   )
    __A	:       Union[str, Any]       =	DatasetMetadata.from_readme(lowercase_   )
    self.assertEqual(lowercase_				,			{}   )



  def 						A_				(self  :		Any   )		->							Tuple:
   __A	:       List[Any]       =	_dedent(
       '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            '   )
   assert DatasetMetadata.from_yaml_string(lowercase_   )

   __A	:       Tuple       =	_dedent(
       '\\n            annotations_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            task_ids:\n            - open-domain-qa\n            '   )
   with self.assertRaises(lowercase_   ):
    DatasetMetadata.from_yaml_string(lowercase_   )

   __A	:       Union[str, Any]       =	_dedent(
       '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - text-classification\n            task_ids:\n            - multi-class-classification\n            paperswithcode_id:\n            - squad\n            configs:\n            - en\n            train-eval-index:\n            - config: en\n              task: text-classification\n              task_id: multi_class_classification\n              splits:\n                train_split: train\n                eval_split: test\n              col_mapping:\n                text: text\n                label: target\n              metrics:\n                - type: accuracy\n                  name: Accuracy\n            extra_gated_prompt: |\n              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:\n              [RESEARCHER_FULLNAME] (the "Researcher") has requested permission to use the ImageNet database (the "Database") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:\n              1. Researcher shall use the Database only for non-commercial research and educational purposes.\n            extra_gated_fields:\n              Company: text\n              Country: text\n              I agree to use this model for non-commerical use ONLY: checkbox\n            '   )
   assert DatasetMetadata.from_yaml_string(lowercase_   )













"""simple docstring"""


import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_download, hf_hub_url
from PIL import Image

from transformers import DetaConfig, DetaForObjectDetection, DetaImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
_UpperCamelCase   :						str     						= logging.get_logger(__name__)




def 				a (     )  ->		int:
 __A	:       Dict       =	DetaConfig(
     num_queries=9_0_0     ,	encoder_ffn_dim=2_0_4_8     ,	decoder_ffn_dim=2_0_4_8     ,	num_feature_levels=5     ,	assign_first_stage=UpperCamelCase_     ,	with_box_refine=UpperCamelCase_     ,	two_stage=UpperCamelCase_     ,	)

 # set labels
 __A	:       List[Any]       =	9_1
 __A	:       Any       =	'huggingface/label-files'
 __A	:       List[Any]       =	'coco-detection-id2label.json'
 __A	:       Union[str, Any]       =	json.load(open(cached_download(hf_hub_url(UpperCamelCase_     ,	UpperCamelCase_     ,	repo_type='dataset'		)		)     ,	'r'		)		)
 __A	:       int       =	{int(UpperCamelCase_		): v for k, v in idalabel.items()}
 __A	:       Optional[int]       =	idalabel
 __A	:       Optional[int]       =	{v: k for k, v in idalabel.items()}

 return config




def 				a (     UpperCamelCase_							:				Dict		)  ->		List[str]:
 __A	:       Optional[int]       =	[]

 # stem
 # fmt: off
 rename_keys.append(('backbone.0.body.conv1.weight', 'model.backbone.model.embedder.embedder.convolution.weight')		)
 rename_keys.append(('backbone.0.body.bn1.weight', 'model.backbone.model.embedder.embedder.normalization.weight')		)
 rename_keys.append(('backbone.0.body.bn1.bias', 'model.backbone.model.embedder.embedder.normalization.bias')		)
 rename_keys.append(('backbone.0.body.bn1.running_mean', 'model.backbone.model.embedder.embedder.normalization.running_mean')		)
 rename_keys.append(('backbone.0.body.bn1.running_var', 'model.backbone.model.embedder.embedder.normalization.running_var')		)
 # stages
 for stage_idx in range(len(config.backbone_config.depths		)		):
  for layer_idx in range(config.backbone_config.depths[stage_idx]		):
   # shortcut
   if layer_idx == 0:
    rename_keys.append(
        (
            F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.0.weight",
            F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.convolution.weight",
        )		)
    rename_keys.append(
        (
            F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.weight",
            F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.weight",
        )		)
    rename_keys.append(
        (
            F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.bias",
            F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.bias",
        )		)
    rename_keys.append(
        (
            F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_mean",
            F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_mean",
        )		)
    rename_keys.append(
        (
            F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_var",
            F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_var",
        )		)
   # 3 convs
   for i in range(3		):
    rename_keys.append(
        (
            F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.conv{i+1}.weight",
            F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.convolution.weight",
        )		)
    rename_keys.append(
        (
            F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.weight",
            F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.weight",
        )		)
    rename_keys.append(
        (
            F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.bias",
            F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.bias",
        )		)
    rename_keys.append(
        (
            F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_mean",
            F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_mean",
        )		)
    rename_keys.append(
        (
            F"backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_var",
            F"model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_var",
        )		)
    # transformer encoder
 for i in range(config.encoder_layers		):
  rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.sampling_offsets.weight", F"model.encoder.layers.{i}.self_attn.sampling_offsets.weight")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.sampling_offsets.bias", F"model.encoder.layers.{i}.self_attn.sampling_offsets.bias")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.attention_weights.weight", F"model.encoder.layers.{i}.self_attn.attention_weights.weight")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.attention_weights.bias", F"model.encoder.layers.{i}.self_attn.attention_weights.bias")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.value_proj.weight", F"model.encoder.layers.{i}.self_attn.value_proj.weight")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.value_proj.bias", F"model.encoder.layers.{i}.self_attn.value_proj.bias")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.output_proj.weight", F"model.encoder.layers.{i}.self_attn.output_proj.weight")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.self_attn.output_proj.bias", F"model.encoder.layers.{i}.self_attn.output_proj.bias")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.norm1.weight", F"model.encoder.layers.{i}.self_attn_layer_norm.weight")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.norm1.bias", F"model.encoder.layers.{i}.self_attn_layer_norm.bias")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.linear1.weight", F"model.encoder.layers.{i}.fc1.weight")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.linear1.bias", F"model.encoder.layers.{i}.fc1.bias")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.linear2.weight", F"model.encoder.layers.{i}.fc2.weight")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.linear2.bias", F"model.encoder.layers.{i}.fc2.bias")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.norm2.weight", F"model.encoder.layers.{i}.final_layer_norm.weight")		)
  rename_keys.append((F"transformer.encoder.layers.{i}.norm2.bias", F"model.encoder.layers.{i}.final_layer_norm.bias")		)

 # transformer decoder
 for i in range(config.decoder_layers		):
  rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.sampling_offsets.weight", F"model.decoder.layers.{i}.encoder_attn.sampling_offsets.weight")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.sampling_offsets.bias", F"model.decoder.layers.{i}.encoder_attn.sampling_offsets.bias")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.attention_weights.weight", F"model.decoder.layers.{i}.encoder_attn.attention_weights.weight")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.attention_weights.bias", F"model.decoder.layers.{i}.encoder_attn.attention_weights.bias")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.value_proj.weight", F"model.decoder.layers.{i}.encoder_attn.value_proj.weight")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.value_proj.bias", F"model.decoder.layers.{i}.encoder_attn.value_proj.bias")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.output_proj.weight", F"model.decoder.layers.{i}.encoder_attn.output_proj.weight")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.cross_attn.output_proj.bias", F"model.decoder.layers.{i}.encoder_attn.output_proj.bias")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.norm1.weight", F"model.decoder.layers.{i}.encoder_attn_layer_norm.weight")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.norm1.bias", F"model.decoder.layers.{i}.encoder_attn_layer_norm.bias")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.self_attn.out_proj.weight", F"model.decoder.layers.{i}.self_attn.out_proj.weight")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.self_attn.out_proj.bias", F"model.decoder.layers.{i}.self_attn.out_proj.bias")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.norm2.weight", F"model.decoder.layers.{i}.self_attn_layer_norm.weight")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.norm2.bias", F"model.decoder.layers.{i}.self_attn_layer_norm.bias")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.linear1.weight", F"model.decoder.layers.{i}.fc1.weight")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.linear1.bias", F"model.decoder.layers.{i}.fc1.bias")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.linear2.weight", F"model.decoder.layers.{i}.fc2.weight")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.linear2.bias", F"model.decoder.layers.{i}.fc2.bias")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.norm3.weight", F"model.decoder.layers.{i}.final_layer_norm.weight")		)
  rename_keys.append((F"transformer.decoder.layers.{i}.norm3.bias", F"model.decoder.layers.{i}.final_layer_norm.bias")		)

 # fmt: on

 return rename_keys




def 				a (     UpperCamelCase_							:				Optional[Any]     ,	UpperCamelCase_							:				List[str]     ,	UpperCamelCase_							:				List[Any]		)  ->		Dict:
 __A	:       List[str]       =	dct.pop(UpperCamelCase_		)
 __A	:       Optional[Any]       =	val




def 				a (     UpperCamelCase_							:				str     ,	UpperCamelCase_							:				List[Any]		)  ->		Union[str, Any]:
 # transformer decoder self-attention layers
 __A	:       Dict       =	config.d_model
 for i in range(config.decoder_layers		):
  # read in weights + bias of input projection layer of self-attention
  __A	:       Any       =	state_dict.pop(F"transformer.decoder.layers.{i}.self_attn.in_proj_weight"		)
  __A	:       Optional[Any]       =	state_dict.pop(F"transformer.decoder.layers.{i}.self_attn.in_proj_bias"		)
  # next, add query, keys and values (in that order) to the state dict
  __A	:       Optional[int]       =	in_proj_weight[:hidden_size, :]
  __A	:       Any       =	in_proj_bias[:hidden_size]
  __A	:       Any       =	in_proj_weight[
      hidden_size : hidden_size * 2, :
  ]
  __A	:       List[Any]       =	in_proj_bias[hidden_size : hidden_size * 2]
  __A	:       Optional[Any]       =	in_proj_weight[-hidden_size:, :]
  __A	:       Optional[Any]       =	in_proj_bias[-hidden_size:]




def 				a (     )  ->		Dict:
 __A	:       str       =	'http://images.cocodataset.org/val2017/000000039769.jpg'
 __A	:       Union[str, Any]       =	Image.open(requests.get(UpperCamelCase_     ,	stream=UpperCamelCase_		).raw		)

 return im




@torch.no_grad()
def 				a (     UpperCamelCase_							:				Union[str, Any]     ,	UpperCamelCase_							:				Tuple     ,	UpperCamelCase_							:				Optional[int]		)  ->		Optional[int]:
 __A	:       Dict       =	get_deta_config()

 # load original state dict
 if model_name == "deta-resnet-50":
  __A	:       str       =	'adet_checkpoint0011.pth'
 elif model_name == "deta-resnet-50-24-epochs":
  __A	:       Dict       =	'adet_2x_checkpoint0023.pth'
 else:
  raise ValueError(F"Model name {model_name} not supported"		)
 __A	:       Tuple       =	hf_hub_download(repo_id='nielsr/deta-checkpoints'     ,	filename=UpperCamelCase_		)
 __A	:       Dict       =	torch.load(UpperCamelCase_     ,	map_location='cpu'		)['model']

 # rename keys
 __A	:       Optional[Any]       =	create_rename_keys(UpperCamelCase_		)
 for src, dest in rename_keys:
  rename_key(UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_		)
 read_in_decoder_q_k_v(UpperCamelCase_     ,	UpperCamelCase_		)

 # fix some prefixes
 for key in state_dict.copy().keys():
  if "transformer.decoder.class_embed" in key or "transformer.decoder.bbox_embed" in key:
   __A	:       Union[str, Any]       =	state_dict.pop(UpperCamelCase_		)
   __A	:       Dict       =	val
  if "input_proj" in key:
   __A	:       List[str]       =	state_dict.pop(UpperCamelCase_		)
   __A	:       Optional[Any]       =	val
  if "level_embed" in key or "pos_trans" in key or "pix_trans" in key or "enc_output" in key:
   __A	:       List[Any]       =	state_dict.pop(UpperCamelCase_		)
   __A	:       Optional[Any]       =	val

    # finally, create HuggingFace model and load state dict
 __A	:       Dict       =	DetaForObjectDetection(UpperCamelCase_		)
 model.load_state_dict(UpperCamelCase_		)
 model.eval()

 __A	:       Any       =	'cuda' if torch.cuda.is_available() else 'cpu'
 model.to(UpperCamelCase_		)

 # load image processor
 __A	:       Any       =	DetaImageProcessor(format='coco_detection'		)

 # verify our conversion on image
 __A	:       str       =	prepare_img()
 __A	:       int       =	processor(images=UpperCamelCase_     ,	return_tensors='pt'		)
 __A	:       str       =	encoding['pixel_values']
 __A	:       Optional[Any]       =	model(pixel_values.to(UpperCamelCase_		)		)

 # verify logits
 if model_name == "deta-resnet-50":
  __A	:       Union[str, Any]       =	torch.tensor(
      [[-7.3978, -2.5406, -4.1668], [-8.2684, -3.9933, -3.8096], [-7.0515, -3.7973, -5.8516]]		)
  __A	:       str       =	torch.tensor([[0.5043, 0.4973, 0.9998], [0.2542, 0.5489, 0.4748], [0.5490, 0.2765, 0.0570]]		)
 elif model_name == "deta-resnet-50-24-epochs":
  __A	:       Tuple       =	torch.tensor(
      [[-7.1688, -2.4857, -4.8669], [-7.8630, -3.8154, -4.2674], [-7.2730, -4.1865, -5.5323]]		)
  __A	:       Optional[int]       =	torch.tensor([[0.5021, 0.4971, 0.9994], [0.2546, 0.5486, 0.4731], [0.1686, 0.1986, 0.2142]]		)

 assert torch.allclose(outputs.logits[0, :3, :3]     ,	expected_logits.to(UpperCamelCase_		)     ,	atol=1E-4		)
 assert torch.allclose(outputs.pred_boxes[0, :3, :3]     ,	expected_boxes.to(UpperCamelCase_		)     ,	atol=1E-4		)
 print('Everything ok!'		)

 if pytorch_dump_folder_path:
  # Save model and processor
  logger.info(F"Saving PyTorch model and processor to {pytorch_dump_folder_path}..."		)
  Path(UpperCamelCase_		).mkdir(exist_ok=UpperCamelCase_		)
  model.save_pretrained(UpperCamelCase_		)
  processor.save_pretrained(UpperCamelCase_		)

 # Push to hub
 if push_to_hub:
  print('Pushing model and processor to hub...'		)
  model.push_to_hub(F"jozhang97/{model_name}"		)
  processor.push_to_hub(F"jozhang97/{model_name}"		)


if __name__ == "__main__":
   _UpperCamelCase   :						Optional[Any]     						= argparse.ArgumentParser()

   parser.add_argument(
       """--model_name""",
       type=str,
       default="""deta-resnet-50""",
       choices=["""deta-resnet-50""", """deta-resnet-50-24-epochs"""],
       help="""Name of the model you'd like to convert.""",
   )
   parser.add_argument(
       """--pytorch_dump_folder_path""",
       default=None,
       type=str,
       help="""Path to the folder to output PyTorch model.""",
   )
   parser.add_argument(
       """--push_to_hub""", action="""store_true""", help="""Whether or not to push the converted model to the 🤗 hub."""
   )
   _UpperCamelCase   :						Tuple     						= parser.parse_args()
   convert_deta_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)





