import argparse
import os

import align
import numpy as np
import requests
import tensorflow as tf
import torch
from PIL import Image
from tokenizer import Tokenizer

from transformers import (
    AlignConfig,
    AlignModel,
    AlignProcessor,
    BertConfig,
    BertTokenizer,
    EfficientNetConfig,
    EfficientNetImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
_SCREAMING_SNAKE_CASE   						=    logging.get_logger(__name__)
def 				__lowercase				(						lowerCamelCase__					)				->str:

  '''simple docstring'''




  a__     		=     tf.image.resize(lowerCamelCase__ ,				(3_46, 3_46)					)
  a__     		=     tf.image.crop_to_bounding_box(lowerCamelCase__ ,				(3_46 - 2_89) // 2 ,				(3_46 - 2_89) // 2 ,				2_89 ,				2_89					)
  return image
def 				__lowercase				(						)				->Union[str, Any]:

  '''simple docstring'''




  a__     		=     EfficientNetConfig.from_pretrained("google/efficientnet-b7"					)
  a__     		=     2_89
  a__     		=     6_40
  a__     		=     {"0": "LABEL_0", "1": "LABEL_1"}
  a__     		=     {"LABEL_0": 0, "LABEL_1": 1}
  a__     		=     []

  a__     		=     BertConfig()
  a__     		=     AlignConfig.from_text_vision_configs(
      text_config=lowerCamelCase__ ,				vision_config=lowerCamelCase__ ,				projection_dim=6_40					)
  return config
def 				__lowercase				(						)				->Optional[Any]:

  '''simple docstring'''




  a__     		=     "http://images.cocodataset.org/val2017/000000039769.jpg"
  a__     		=     Image.open(requests.get(lowerCamelCase__ ,				stream=lowerCamelCase__					).raw					)
  return im
def 				__lowercase				(						)				->List[Any]:

  '''simple docstring'''




  a__     		=     EfficientNetImageProcessor(
      do_center_crop=lowerCamelCase__ ,				rescale_factor=1 / 127.5 ,				rescale_offset=lowerCamelCase__ ,				do_normalize=lowerCamelCase__ ,				include_top=lowerCamelCase__ ,				resample=Image.BILINEAR ,				)
  a__     		=     BertTokenizer.from_pretrained("bert-base-uncased"					)
  a__     		=     64
  a__     		=     AlignProcessor(image_processor=lowerCamelCase__ ,				tokenizer=lowerCamelCase__					)
  return processor
def 				__lowercase				(						lowerCamelCase__					)				->Union[str, Any]:

  '''simple docstring'''




  a__     		=     [v.split("_"					)[0].split("block"					)[1] for v in original_param_names if v.startswith("block"					)]
  a__     		=     list(set(lowerCamelCase__					)					)
  a__     		=     sorted(lowerCamelCase__					)
  a__     		=     len(lowerCamelCase__					)
  a__     		=     {b: str(lowerCamelCase__					) for b, i in zip(lowerCamelCase__ ,				range(lowerCamelCase__					)					)}

  a__     		=     []
  rename_keys.append(("stem_conv/kernel:0", "embeddings.convolution.weight")					)
  rename_keys.append(("stem_bn/gamma:0", "embeddings.batchnorm.weight")					)
  rename_keys.append(("stem_bn/beta:0", "embeddings.batchnorm.bias")					)
  rename_keys.append(("stem_bn/moving_mean:0", "embeddings.batchnorm.running_mean")					)
  rename_keys.append(("stem_bn/moving_variance:0", "embeddings.batchnorm.running_var")					)

  for b in block_names:
    a__     		=     block_name_mapping[b]
    rename_keys.append((f"block{b}_expand_conv/kernel:0", f"encoder.blocks.{hf_b}.expansion.expand_conv.weight")					)
    rename_keys.append((f"block{b}_expand_bn/gamma:0", f"encoder.blocks.{hf_b}.expansion.expand_bn.weight")					)
    rename_keys.append((f"block{b}_expand_bn/beta:0", f"encoder.blocks.{hf_b}.expansion.expand_bn.bias")					)
    rename_keys.append(
        (f"block{b}_expand_bn/moving_mean:0", f"encoder.blocks.{hf_b}.expansion.expand_bn.running_mean")					)
    rename_keys.append(
        (f"block{b}_expand_bn/moving_variance:0", f"encoder.blocks.{hf_b}.expansion.expand_bn.running_var")					)
    rename_keys.append(
        (f"block{b}_dwconv/depthwise_kernel:0", f"encoder.blocks.{hf_b}.depthwise_conv.depthwise_conv.weight")					)
    rename_keys.append((f"block{b}_bn/gamma:0", f"encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.weight")					)
    rename_keys.append((f"block{b}_bn/beta:0", f"encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.bias")					)
    rename_keys.append(
        (f"block{b}_bn/moving_mean:0", f"encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.running_mean")					)
    rename_keys.append(
        (f"block{b}_bn/moving_variance:0", f"encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.running_var")					)

    rename_keys.append((f"block{b}_se_reduce/kernel:0", f"encoder.blocks.{hf_b}.squeeze_excite.reduce.weight")					)
    rename_keys.append((f"block{b}_se_reduce/bias:0", f"encoder.blocks.{hf_b}.squeeze_excite.reduce.bias")					)
    rename_keys.append((f"block{b}_se_expand/kernel:0", f"encoder.blocks.{hf_b}.squeeze_excite.expand.weight")					)
    rename_keys.append((f"block{b}_se_expand/bias:0", f"encoder.blocks.{hf_b}.squeeze_excite.expand.bias")					)
    rename_keys.append(
        (f"block{b}_project_conv/kernel:0", f"encoder.blocks.{hf_b}.projection.project_conv.weight")					)
    rename_keys.append((f"block{b}_project_bn/gamma:0", f"encoder.blocks.{hf_b}.projection.project_bn.weight")					)
    rename_keys.append((f"block{b}_project_bn/beta:0", f"encoder.blocks.{hf_b}.projection.project_bn.bias")					)
    rename_keys.append(
        (f"block{b}_project_bn/moving_mean:0", f"encoder.blocks.{hf_b}.projection.project_bn.running_mean")					)
    rename_keys.append(
        (f"block{b}_project_bn/moving_variance:0", f"encoder.blocks.{hf_b}.projection.project_bn.running_var")					)

  a__     		=     {}
  for item in rename_keys:
    if item[0] in original_param_names:
      a__     		=     "vision_model." + item[1]

    # BERT text encoder
  a__     		=     []
  a__     		=     "tf_bert_model/bert"
  a__     		=     "text_model"
  for i in range(12					):
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/attention/self/query/kernel:0",
            f"{new}.encoder.layer.{i}.attention.self.query.weight",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/attention/self/query/bias:0",
            f"{new}.encoder.layer.{i}.attention.self.query.bias",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/attention/self/key/kernel:0",
            f"{new}.encoder.layer.{i}.attention.self.key.weight",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/attention/self/key/bias:0",
            f"{new}.encoder.layer.{i}.attention.self.key.bias",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/attention/self/value/kernel:0",
            f"{new}.encoder.layer.{i}.attention.self.value.weight",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/attention/self/value/bias:0",
            f"{new}.encoder.layer.{i}.attention.self.value.bias",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/attention/output/dense/kernel:0",
            f"{new}.encoder.layer.{i}.attention.output.dense.weight",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/attention/output/dense/bias:0",
            f"{new}.encoder.layer.{i}.attention.output.dense.bias",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/attention/output/LayerNorm/gamma:0",
            f"{new}.encoder.layer.{i}.attention.output.LayerNorm.weight",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/attention/output/LayerNorm/beta:0",
            f"{new}.encoder.layer.{i}.attention.output.LayerNorm.bias",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/intermediate/dense/kernel:0",
            f"{new}.encoder.layer.{i}.intermediate.dense.weight",
        )					)
    rename_keys.append(
        (
            f"{old}/encoder/layer_._{i}/intermediate/dense/bias:0",
            f"{new}.encoder.layer.{i}.intermediate.dense.bias",
        )					)
    rename_keys.append(
        (f"{old}/encoder/layer_._{i}/output/dense/kernel:0", f"{new}.encoder.layer.{i}.output.dense.weight")					)
    rename_keys.append(
        (f"{old}/encoder/layer_._{i}/output/dense/bias:0", f"{new}.encoder.layer.{i}.output.dense.bias")					)
    rename_keys.append(
        (f"{old}/encoder/layer_._{i}/output/LayerNorm/gamma:0", f"{new}.encoder.layer.{i}.output.LayerNorm.weight")					)
    rename_keys.append(
        (f"{old}/encoder/layer_._{i}/output/LayerNorm/beta:0", f"{new}.encoder.layer.{i}.output.LayerNorm.bias")					)

  rename_keys.append((f"{old}/embeddings/word_embeddings/weight:0", f"{new}.embeddings.word_embeddings.weight")					)
  rename_keys.append(
      (f"{old}/embeddings/position_embeddings/embeddings:0", f"{new}.embeddings.position_embeddings.weight")					)
  rename_keys.append(
      (f"{old}/embeddings/token_type_embeddings/embeddings:0", f"{new}.embeddings.token_type_embeddings.weight")					)
  rename_keys.append((f"{old}/embeddings/LayerNorm/gamma:0", f"{new}.embeddings.LayerNorm.weight")					)
  rename_keys.append((f"{old}/embeddings/LayerNorm/beta:0", f"{new}.embeddings.LayerNorm.bias")					)

  rename_keys.append((f"{old}/pooler/dense/kernel:0", f"{new}.pooler.dense.weight")					)
  rename_keys.append((f"{old}/pooler/dense/bias:0", f"{new}.pooler.dense.bias")					)
  rename_keys.append(("dense/kernel:0", "text_projection.weight")					)
  rename_keys.append(("dense/bias:0", "text_projection.bias")					)
  rename_keys.append(("dense/bias:0", "text_projection.bias")					)
  rename_keys.append(("temperature:0", "temperature")					)

  for item in rename_keys:
    if item[0] in original_param_names:
      a__     		=     item[1]
  return key_mapping
def 				__lowercase				(						lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					)				->Optional[Any]:

  '''simple docstring'''




  list(hf_params.keys()					)

  for key, value in tf_params.items():
    if key not in key_mapping:
      continue

    a__     		=     key_mapping[key]
    if "_conv" in key and "kernel" in key:
      a__     		=     torch.from_numpy(lowerCamelCase__					).permute(3 ,				2 ,				0 ,				1					)
    elif "embeddings" in key:
      a__     		=     torch.from_numpy(lowerCamelCase__					)
    elif "depthwise_kernel" in key:
      a__     		=     torch.from_numpy(lowerCamelCase__					).permute(2 ,				3 ,				0 ,				1					)
    elif "kernel" in key:
      a__     		=     torch.from_numpy(np.transpose(lowerCamelCase__					)					)
    elif "temperature" in key:
      a__     		=     value
    elif "bn/gamma" or "bn/beta" in key:
      a__     		=     torch.from_numpy(np.transpose(lowerCamelCase__					)					).squeeze()
    else:
      a__     		=     torch.from_numpy(lowerCamelCase__					)

    # Replace HF parameters with original TF model parameters
    hf_params[hf_key].copy_(lowerCamelCase__					)
@torch.no_grad()
def 				__lowercase				(						lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					)				->Optional[int]:

  '''simple docstring'''




  a__     		=     64
  a__     		=     Tokenizer(lowerCamelCase__					)
  a__     		=     align.Align("efficientnet-b7" ,				"bert-base" ,				6_40 ,				lowerCamelCase__ ,				tok.get_vocab_size()					)
  original_model.compile()
  original_model.load_weights(lowerCamelCase__					)

  a__     		=     original_model.trainable_variables
  a__     		=     original_model.non_trainable_variables
  a__     		=     {param.name: param.numpy() for param in tf_params}
  for param in tf_non_train_params:
    a__     		=     param.numpy()
  a__     		=     list(tf_params.keys()					)

  # Load HuggingFace model
  a__     		=     get_align_config()
  a__     		=     AlignModel(lowerCamelCase__					).eval()
  a__     		=     hf_model.state_dict()

  # Create src-to-dst parameter name mapping dictionary
  print("Converting parameters..."					)
  a__     		=     rename_keys(lowerCamelCase__					)
  replace_params(lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					)

  # Initialize processor
  a__     		=     get_processor()
  a__     		=     processor(
      images=prepare_img() ,				text="A picture of a cat" ,				padding="max_length" ,				max_length=64 ,				return_tensors="pt"					)

  # HF model inference
  hf_model.eval()
  with torch.no_grad():
    a__     		=     hf_model(**lowerCamelCase__					)

  a__     		=     outputs.image_embeds.detach().numpy()
  a__     		=     outputs.text_embeds.detach().numpy()

  # Original model inference
  a__     		=     False
  a__     		=     EfficientNetImageProcessor(
      do_center_crop=lowerCamelCase__ ,				do_rescale=lowerCamelCase__ ,				do_normalize=lowerCamelCase__ ,				include_top=lowerCamelCase__ ,				resample=Image.BILINEAR ,				)
  a__     		=     tf_image_processor(images=prepare_img() ,				return_tensors="tf" ,				data_format="channels_last"					)["pixel_values"]
  a__     		=     tok(tf.constant(["A picture of a cat"]					)					)

  a__     		=     original_model.image_encoder(lowerCamelCase__ ,				training=lowerCamelCase__					)
  a__     		=     original_model.text_encoder(lowerCamelCase__ ,				training=lowerCamelCase__					)

  a__     		=     tf.nn.la_normalize(lowerCamelCase__ ,				axis=-1					)
  a__     		=     tf.nn.la_normalize(lowerCamelCase__ ,				axis=-1					)

  # Check whether original and HF model outputs match  -> np.allclose
  if not np.allclose(lowerCamelCase__ ,				lowerCamelCase__ ,				atol=1E-3					):
    raise ValueError("The predicted image features are not the same."					)
  if not np.allclose(lowerCamelCase__ ,				lowerCamelCase__ ,				atol=1E-3					):
    raise ValueError("The predicted text features are not the same."					)
  print("Model outputs match!"					)

  if save_model:
    # Create folder to save model
    if not os.path.isdir(lowerCamelCase__					):
      os.mkdir(lowerCamelCase__					)
    # Save converted model and image processor
    hf_model.save_pretrained(lowerCamelCase__					)
    processor.save_pretrained(lowerCamelCase__					)

  if push_to_hub:
    # Push model and image processor to hub
    print("Pushing converted ALIGN to the hub..."					)
    processor.push_to_hub("align-base"					)
    hf_model.push_to_hub("align-base"					)


if __name__ == "__main__":
     _SCREAMING_SNAKE_CASE   						=    argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         """--checkpoint_path""",
         default="""./weights/model-weights""",
         type=str,
         help="""Path to the pretrained TF ALIGN checkpoint.""",
     )
     parser.add_argument(
         """--pytorch_dump_folder_path""",
         default="""hf_model""",
         type=str,
         help="""Path to the output PyTorch model directory.""",
     )
     parser.add_argument("""--save_model""", action="""store_true""", help="""Save model to local""")
     parser.add_argument("""--push_to_hub""", action="""store_true""", help="""Push model and image processor to the hub""")

     _SCREAMING_SNAKE_CASE   						=    parser.parse_args()
     convert_align_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.save_model, args.push_to_hub)


# This model implementation is heavily based on:

import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    ControlNetModel,
    DDIMScheduler,
    StableDiffusionControlNetInpaintPipeline,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet import MultiControlNetModel
from diffusers.utils import floats_tensor, load_image, load_numpy, randn_tensor, slow, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_PARAMS,
    TEXT_TO_IMAGE_IMAGE_PARAMS,
)
from ..test_pipelines_common import (
    PipelineKarrasSchedulerTesterMixin,
    PipelineLatentTesterMixin,
    PipelineTesterMixin,
)


enable_full_determinism()







class 					UpperCAmelCase_				(							_a     ,     _a     ,     _a     ,     unittest.TestCase    ):
    UpperCAmelCase_			:       Dict						 =							StableDiffusionControlNetInpaintPipeline
    UpperCAmelCase_			:       Tuple						 =							TEXT_GUIDED_IMAGE_INPAINTING_PARAMS
    UpperCAmelCase_			:       Optional[Any]						 =							TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS
    UpperCAmelCase_			:       Any						 =							frozenset({'control_image'}    )  # skip `image` and `mask` for now, only test for control_image
    UpperCAmelCase_			:       str						 =							TEXT_TO_IMAGE_IMAGE_PARAMS




    def 						__lowerCamelCase (					self  :   Tuple   )	-> Union[str, Any]:
      torch.manual_seed(0   )
      a__     		=     UNetaDConditionModel(
          block_out_channels=(32, 64)      ,      layers_per_block=2      ,      sample_size=32      ,      in_channels=9      ,      out_channels=4      ,      down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")      ,      up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")      ,      cross_attention_dim=32      ,      )
      torch.manual_seed(0   )
      a__     		=     ControlNetModel(
          block_out_channels=(32, 64)      ,      layers_per_block=2      ,      in_channels=4      ,      down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")      ,      cross_attention_dim=32      ,      conditioning_embedding_out_channels=(16, 32)      ,      )
      torch.manual_seed(0   )
      a__     		=     DDIMScheduler(
          beta_start=0.00_085      ,      beta_end=0.012      ,      beta_schedule="scaled_linear"      ,      clip_sample=lowerCAmelCase      ,      set_alpha_to_one=lowerCAmelCase      ,      )
      torch.manual_seed(0   )
      a__     		=     AutoencoderKL(
          block_out_channels=[32, 64]      ,      in_channels=3      ,      out_channels=3      ,      down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]      ,      up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]      ,      latent_channels=4      ,      )
      torch.manual_seed(0   )
      a__     		=     CLIPTextConfig(
          bos_token_id=0      ,      eos_token_id=2      ,      hidden_size=32      ,      intermediate_size=37      ,      layer_norm_eps=1e-0_5      ,      num_attention_heads=4      ,      num_hidden_layers=5      ,      pad_token_id=1      ,      vocab_size=1000      ,      )
      a__     		=     CLIPTextModel(lowerCAmelCase   )
      a__     		=     CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"   )

      a__     		=     {
          "unet": unet,
          "controlnet": controlnet,
          "scheduler": scheduler,
          "vae": vae,
          "text_encoder": text_encoder,
          "tokenizer": tokenizer,
          "safety_checker": None,
          "feature_extractor": None,
      }
      return components




    def 						__lowerCamelCase (					self  :   Tuple      ,      lowerCAmelCase  :   Optional[Any]      ,      lowerCAmelCase  :   Dict=0   )	-> Optional[Any]:
      if str(lowerCAmelCase   ).startswith("mps"   ):
        a__     		=     torch.manual_seed(lowerCAmelCase   )
      else:
        a__     		=     torch.Generator(device=lowerCAmelCase   ).manual_seed(lowerCAmelCase   )

      a__     		=     2
      a__     		=     randn_tensor(
          (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)      ,      generator=lowerCAmelCase      ,      device=torch.device(lowerCAmelCase   )      ,      )
      a__     		=     floats_tensor((1, 3, 32, 32)      ,      rng=random.Random(lowerCAmelCase   )   ).to(lowerCAmelCase   )
      a__     		=     init_image.cpu().permute(0      ,      2      ,      3      ,      1   )[0]

      a__     		=     Image.fromarray(np.uinta(lowerCAmelCase   )   ).convert("RGB"   ).resize((64, 64)   )
      a__     		=     Image.fromarray(np.uinta(init_image + 4   )   ).convert("RGB"   ).resize((64, 64)   )

      a__     		=     {
          "prompt": "A painting of a squirrel eating a burger",
          "generator": generator,
          "num_inference_steps": 2,
          "guidance_scale": 6.0,
          "output_type": "numpy",
          "image": image,
          "mask_image": mask_image,
          "control_image": control_image,
      }

      return inputs




    def 						__lowerCamelCase (					self  :   Any   )	-> Tuple:
      return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3   )




    @unittest.skipIf(
        torch_device != "cuda" or not is_xformers_available()      ,      reason="XFormers attention is only available with CUDA and `xformers` installed"      ,      )
    def 						__lowerCamelCase (					self  :   List[Any]   )	-> Optional[int]:
      self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3   )




    def 						__lowerCamelCase (					self  :   Any   )	-> Optional[int]:
      self._test_inference_batch_single_identical(expected_max_diff=2e-3   )







class 					UpperCAmelCase_				(							_a    ):
    UpperCAmelCase_			:       Tuple						 =							StableDiffusionControlNetInpaintPipeline
    UpperCAmelCase_			:       List[Any]						 =							TEXT_GUIDED_IMAGE_INPAINTING_PARAMS
    UpperCAmelCase_			:       Dict						 =							TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS
    UpperCAmelCase_			:       Tuple						 =							frozenset([]    )




    def 						__lowerCamelCase (					self  :   int   )	-> Dict:
      torch.manual_seed(0   )
      a__     		=     UNetaDConditionModel(
          block_out_channels=(32, 64)      ,      layers_per_block=2      ,      sample_size=32      ,      in_channels=4      ,      out_channels=4      ,      down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")      ,      up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")      ,      cross_attention_dim=32      ,      )
      torch.manual_seed(0   )
      a__     		=     ControlNetModel(
          block_out_channels=(32, 64)      ,      layers_per_block=2      ,      in_channels=4      ,      down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")      ,      cross_attention_dim=32      ,      conditioning_embedding_out_channels=(16, 32)      ,      )
      torch.manual_seed(0   )
      a__     		=     DDIMScheduler(
          beta_start=0.00_085      ,      beta_end=0.012      ,      beta_schedule="scaled_linear"      ,      clip_sample=lowerCAmelCase      ,      set_alpha_to_one=lowerCAmelCase      ,      )
      torch.manual_seed(0   )
      a__     		=     AutoencoderKL(
          block_out_channels=[32, 64]      ,      in_channels=3      ,      out_channels=3      ,      down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]      ,      up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]      ,      latent_channels=4      ,      )
      torch.manual_seed(0   )
      a__     		=     CLIPTextConfig(
          bos_token_id=0      ,      eos_token_id=2      ,      hidden_size=32      ,      intermediate_size=37      ,      layer_norm_eps=1e-0_5      ,      num_attention_heads=4      ,      num_hidden_layers=5      ,      pad_token_id=1      ,      vocab_size=1000      ,      )
      a__     		=     CLIPTextModel(lowerCAmelCase   )
      a__     		=     CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"   )

      a__     		=     {
          "unet": unet,
          "controlnet": controlnet,
          "scheduler": scheduler,
          "vae": vae,
          "text_encoder": text_encoder,
          "tokenizer": tokenizer,
          "safety_checker": None,
          "feature_extractor": None,
      }
      return components







class 					UpperCAmelCase_				(							_a     ,     _a     ,     unittest.TestCase    ):
    UpperCAmelCase_			:       Union[str, Any]						 =							StableDiffusionControlNetInpaintPipeline
    UpperCAmelCase_			:       Any						 =							TEXT_GUIDED_IMAGE_INPAINTING_PARAMS
    UpperCAmelCase_			:       Optional[int]						 =							TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS




    def 						__lowerCamelCase (					self  :   Dict   )	-> str:
      torch.manual_seed(0   )
      a__     		=     UNetaDConditionModel(
          block_out_channels=(32, 64)      ,      layers_per_block=2      ,      sample_size=32      ,      in_channels=9      ,      out_channels=4      ,      down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")      ,      up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")      ,      cross_attention_dim=32      ,      )
      torch.manual_seed(0   )

      def init_weights(lowerCAmelCase  :   Optional[int]   ):
        if isinstance(lowerCAmelCase      ,      torch.nn.Convad   ):
          torch.nn.init.normal(m.weight   )
          m.bias.data.fill_(1.0   )

      a__     		=     ControlNetModel(
          block_out_channels=(32, 64)      ,      layers_per_block=2      ,      in_channels=4      ,      down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")      ,      cross_attention_dim=32      ,      conditioning_embedding_out_channels=(16, 32)      ,      )
      controlneta.controlnet_down_blocks.apply(lowerCAmelCase   )

      torch.manual_seed(0   )
      a__     		=     ControlNetModel(
          block_out_channels=(32, 64)      ,      layers_per_block=2      ,      in_channels=4      ,      down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")      ,      cross_attention_dim=32      ,      conditioning_embedding_out_channels=(16, 32)      ,      )
      controlneta.controlnet_down_blocks.apply(lowerCAmelCase   )

      torch.manual_seed(0   )
      a__     		=     DDIMScheduler(
          beta_start=0.00_085      ,      beta_end=0.012      ,      beta_schedule="scaled_linear"      ,      clip_sample=lowerCAmelCase      ,      set_alpha_to_one=lowerCAmelCase      ,      )
      torch.manual_seed(0   )
      a__     		=     AutoencoderKL(
          block_out_channels=[32, 64]      ,      in_channels=3      ,      out_channels=3      ,      down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]      ,      up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]      ,      latent_channels=4      ,      )
      torch.manual_seed(0   )
      a__     		=     CLIPTextConfig(
          bos_token_id=0      ,      eos_token_id=2      ,      hidden_size=32      ,      intermediate_size=37      ,      layer_norm_eps=1e-0_5      ,      num_attention_heads=4      ,      num_hidden_layers=5      ,      pad_token_id=1      ,      vocab_size=1000      ,      )
      a__     		=     CLIPTextModel(lowerCAmelCase   )
      a__     		=     CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"   )

      a__     		=     MultiControlNetModel([controlneta, controlneta]   )

      a__     		=     {
          "unet": unet,
          "controlnet": controlnet,
          "scheduler": scheduler,
          "vae": vae,
          "text_encoder": text_encoder,
          "tokenizer": tokenizer,
          "safety_checker": None,
          "feature_extractor": None,
      }
      return components




    def 						__lowerCamelCase (					self  :   Optional[int]      ,      lowerCAmelCase  :   Dict      ,      lowerCAmelCase  :   Any=0   )	-> List[str]:
      if str(lowerCAmelCase   ).startswith("mps"   ):
        a__     		=     torch.manual_seed(lowerCAmelCase   )
      else:
        a__     		=     torch.Generator(device=lowerCAmelCase   ).manual_seed(lowerCAmelCase   )

      a__     		=     2

      a__     		=     [
          randn_tensor(
              (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)      ,      generator=lowerCAmelCase      ,      device=torch.device(lowerCAmelCase   )      ,      ),
          randn_tensor(
              (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)      ,      generator=lowerCAmelCase      ,      device=torch.device(lowerCAmelCase   )      ,      ),
      ]
      a__     		=     floats_tensor((1, 3, 32, 32)      ,      rng=random.Random(lowerCAmelCase   )   ).to(lowerCAmelCase   )
      a__     		=     init_image.cpu().permute(0      ,      2      ,      3      ,      1   )[0]

      a__     		=     Image.fromarray(np.uinta(lowerCAmelCase   )   ).convert("RGB"   ).resize((64, 64)   )
      a__     		=     Image.fromarray(np.uinta(init_image + 4   )   ).convert("RGB"   ).resize((64, 64)   )

      a__     		=     {
          "prompt": "A painting of a squirrel eating a burger",
          "generator": generator,
          "num_inference_steps": 2,
          "guidance_scale": 6.0,
          "output_type": "numpy",
          "image": image,
          "mask_image": mask_image,
          "control_image": control_image,
      }

      return inputs




    def 						__lowerCamelCase (					self  :   Optional[Any]   )	-> List[Any]:
      a__     		=     self.get_dummy_components()
      a__     		=     self.pipeline_class(**lowerCAmelCase   )
      pipe.to(lowerCAmelCase   )

      a__     		=     10.0
      a__     		=     4

      a__     		=     self.get_dummy_inputs(lowerCAmelCase   )
      a__     		=     steps
      a__     		=     scale
      a__     		=     pipe(**lowerCAmelCase   )[0]

      a__     		=     self.get_dummy_inputs(lowerCAmelCase   )
      a__     		=     steps
      a__     		=     scale
      a__     		=     pipe(**lowerCAmelCase      ,      control_guidance_start=0.1      ,      control_guidance_end=0.2   )[0]

      a__     		=     self.get_dummy_inputs(lowerCAmelCase   )
      a__     		=     steps
      a__     		=     scale
      a__     		=     pipe(**lowerCAmelCase      ,      control_guidance_start=[0.1, 0.3]      ,      control_guidance_end=[0.2, 0.7]   )[0]

      a__     		=     self.get_dummy_inputs(lowerCAmelCase   )
      a__     		=     steps
      a__     		=     scale
      a__     		=     pipe(**lowerCAmelCase      ,      control_guidance_start=0.4      ,      control_guidance_end=[0.5, 0.8]   )[0]

      # make sure that all outputs are different
      assert np.sum(np.abs(output_a - output_a   )   ) > 1e-3
      assert np.sum(np.abs(output_a - output_a   )   ) > 1e-3
      assert np.sum(np.abs(output_a - output_a   )   ) > 1e-3




    def 						__lowerCamelCase (					self  :   Tuple   )	-> Any:
      return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3   )




    @unittest.skipIf(
        torch_device != "cuda" or not is_xformers_available()      ,      reason="XFormers attention is only available with CUDA and `xformers` installed"      ,      )
    def 						__lowerCamelCase (					self  :   Any   )	-> Dict:
      self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3   )




    def 						__lowerCamelCase (					self  :   Any   )	-> List[str]:
      self._test_inference_batch_single_identical(expected_max_diff=2e-3   )




    def 						__lowerCamelCase (					self  :   List[str]   )	-> Any:
      a__     		=     self.get_dummy_components()
      a__     		=     self.pipeline_class(**lowerCAmelCase   )
      pipe.to(lowerCAmelCase   )
      pipe.set_progress_bar_config(disable=lowerCAmelCase   )
      with tempfile.TemporaryDirectory() as tmpdir:
        try:
          # save_pretrained is not implemented for Multi-ControlNet
          pipe.save_pretrained(lowerCAmelCase   )
        except NotImplementedError:
          pass







@slow
@require_torch_gpu
class 					UpperCAmelCase_				(							unittest.TestCase    ):




    def 						__lowerCamelCase (					self  :   List[str]   )	-> Tuple:
      super().tearDown()
      gc.collect()
      torch.cuda.empty_cache()




    def 						__lowerCamelCase (					self  :   Union[str, Any]   )	-> List[Any]:
      a__     		=     ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny"   )

      a__     		=     StableDiffusionControlNetInpaintPipeline.from_pretrained(
          "runwayml/stable-diffusion-inpainting"      ,      safety_checker=lowerCAmelCase      ,      controlnet=lowerCAmelCase   )
      pipe.enable_model_cpu_offload()
      pipe.set_progress_bar_config(disable=lowerCAmelCase   )

      a__     		=     torch.Generator(device="cpu"   ).manual_seed(0   )
      a__     		=     load_image(
          "https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png"   ).resize((512, 512)   )

      a__     		=     load_image(
          "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main"
          "/stable_diffusion_inpaint/input_bench_mask.png"   ).resize((512, 512)   )

      a__     		=     "pitch black hole"

      a__     		=     load_image(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png"   ).resize((512, 512)   )

      a__     		=     pipe(
          lowerCAmelCase      ,      image=lowerCAmelCase      ,      mask_image=lowerCAmelCase      ,      control_image=lowerCAmelCase      ,      generator=lowerCAmelCase      ,      output_type="np"      ,      num_inference_steps=3      ,      )

      a__     		=     output.images[0]

      assert image.shape == (512, 512, 3)

      a__     		=     load_numpy(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/inpaint.npy"   )

      assert np.abs(expected_image - image   ).max() < 9e-2




    def 						__lowerCamelCase (					self  :   Dict   )	-> List[str]:
      a__     		=     ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_inpaint"   )

      a__     		=     StableDiffusionControlNetInpaintPipeline.from_pretrained(
          "runwayml/stable-diffusion-v1-5"      ,      safety_checker=lowerCAmelCase      ,      controlnet=lowerCAmelCase   )
      a__     		=     DDIMScheduler.from_config(pipe.scheduler.config   )
      pipe.enable_model_cpu_offload()
      pipe.set_progress_bar_config(disable=lowerCAmelCase   )

      a__     		=     torch.Generator(device="cpu"   ).manual_seed(33   )

      a__     		=     load_image(
          "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy.png"   )
      a__     		=     init_image.resize((512, 512)   )

      a__     		=     load_image(
          "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy_mask.png"   )
      a__     		=     mask_image.resize((512, 512)   )

      a__     		=     "a handsome man with ray-ban sunglasses"

      def make_inpaint_condition(lowerCAmelCase  :   Any      ,      lowerCAmelCase  :   Optional[int]   ):
        a__     		=     np.array(image.convert("RGB"   )   ).astype(np.floataa   ) / 255.0
        a__     		=     np.array(image_mask.convert("L"   )   ).astype(np.floataa   ) / 255.0

        assert image.shape[0:1] == image_mask.shape[0:1], "image and image_mask must have the same image size"
        a__     		=     -1.0  # set as masked pixel
        a__     		=     np.expand_dims(lowerCAmelCase      ,      0   ).transpose(0      ,      3      ,      1      ,      2   )
        a__     		=     torch.from_numpy(lowerCAmelCase   )
        return image

      a__     		=     make_inpaint_condition(lowerCAmelCase      ,      lowerCAmelCase   )

      a__     		=     pipe(
          lowerCAmelCase      ,      image=lowerCAmelCase      ,      mask_image=lowerCAmelCase      ,      control_image=lowerCAmelCase      ,      guidance_scale=9.0      ,      eta=1.0      ,      generator=lowerCAmelCase      ,      num_inference_steps=20      ,      output_type="np"      ,      )
      a__     		=     output.images[0]

      assert image.shape == (512, 512, 3)

      a__     		=     load_numpy(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/boy_ray_ban.npy"   )

      assert np.abs(expected_image - image   ).max() < 9e-2

