





'''simple docstring'''
import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.semantic_stable_diffusion import SemanticStableDiffusionPipeline as StableDiffusionPipeline
from diffusers.utils import floats_tensor, nightly, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()


class     _lowerCAmelCase					(					unittest.TestCase   ):







							'''simple docstring'''

							def  lowercase							(self						)  ->     Tuple:
														# clean up the VRAM after each test
														super().tearDown()
														gc.collect()
														torch.cuda.empty_cache()

							@property
							def  lowercase							(self						)  ->     Optional[Any]:
														_snake_case     =						1
														_snake_case     =						3
														_snake_case     =						(32, 32)

														_snake_case     =						floats_tensor((batch_size, num_channels) + sizes						,						rng=random.Random(0						)						).to(UpperCAmelCase						)
														return image

							@property
							def  lowercase							(self						)  ->     Optional[int]:
														torch.manual_seed(0						)
														_snake_case     =						UNetaDConditionModel(
														    block_out_channels=(32, 64)						,						layers_per_block=2						,						sample_size=32						,						in_channels=4						,						out_channels=4						,						down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")						,						up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")						,						cross_attention_dim=32						,						)
														return model

							@property
							def  lowercase							(self						)  ->     List[Any]:
														torch.manual_seed(0						)
														_snake_case     =						AutoencoderKL(
														    block_out_channels=[32, 64]						,						in_channels=3						,						out_channels=3						,						down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]						,						up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]						,						latent_channels=4						,						)
														return model

							@property
							def  lowercase							(self						)  ->     int:
														torch.manual_seed(0						)
														_snake_case     =						CLIPTextConfig(
														    bos_token_id=0						,						eos_token_id=2						,						hidden_size=32						,						intermediate_size=37						,						layer_norm_eps=1e-0_5						,						num_attention_heads=4						,						num_hidden_layers=5						,						pad_token_id=1						,						vocab_size=1000						,						)
														return CLIPTextModel(UpperCAmelCase						)

							@property
							def  lowercase							(self						)  ->     List[Any]:
														def extract(*UpperCAmelCase						,						**UpperCAmelCase						):


																					class     _lowerCAmelCase					:







																												'''simple docstring'''

																												def __init__(self						)  ->     Dict:
																																			_snake_case     =						torch.ones([0]						)





																												def  lowercase							(self						,						UpperCAmelCase						)  ->     List[str]:
																																			self.pixel_values.to(UpperCAmelCase						)
																																			return self
																					return Out()

														return extract

							def  lowercase							(self						)  ->     str:
														_snake_case     =						"""cpu"""  # ensure determinism for the device-dependent torch.Generator
														_snake_case     =						self.dummy_cond_unet
														_snake_case     =						DDIMScheduler(
														    beta_start=0.0_0085						,						beta_end=0.012						,						beta_schedule="""scaled_linear"""						,						clip_sample=UpperCAmelCase						,						set_alpha_to_one=UpperCAmelCase						,						)

														_snake_case     =						self.dummy_vae
														_snake_case     =						self.dummy_text_encoder
														_snake_case     =						CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""						)

														# make sure here that pndm scheduler skips prk
														_snake_case     =						StableDiffusionPipeline(
														    unet=UpperCAmelCase						,						scheduler=UpperCAmelCase						,						vae=UpperCAmelCase						,						text_encoder=UpperCAmelCase						,						tokenizer=UpperCAmelCase						,						safety_checker=UpperCAmelCase						,						feature_extractor=self.dummy_extractor						,						)
														_snake_case     =						sd_pipe.to(UpperCAmelCase						)
														sd_pipe.set_progress_bar_config(disable=UpperCAmelCase						)

														_snake_case     =						"""A painting of a squirrel eating a burger"""

														_snake_case     =						torch.Generator(device=UpperCAmelCase						).manual_seed(0						)
														_snake_case     =						sd_pipe([prompt]						,						generator=UpperCAmelCase						,						guidance_scale=6.0						,						num_inference_steps=2						,						output_type="""np"""						)
														_snake_case     =						output.images

														_snake_case     =						torch.Generator(device=UpperCAmelCase						).manual_seed(0						)
														_snake_case     =						sd_pipe(
														    [prompt]						,						generator=UpperCAmelCase						,						guidance_scale=6.0						,						num_inference_steps=2						,						output_type="""np"""						,						return_dict=UpperCAmelCase						,						)[0]

														_snake_case     =						image[0, -3:, -3:, -1]
														_snake_case     =						image_from_tuple[0, -3:, -3:, -1]

														assert image.shape == (1, 64, 64, 3)
														_snake_case     =						np.array([0.5753, 0.6114, 0.5001, 0.5034, 0.5470, 0.4729, 0.4971, 0.4867, 0.4867]						)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2
														assert np.abs(image_from_tuple_slice.flatten() - expected_slice						).max() < 1e-2

							def  lowercase							(self						)  ->     Tuple:
														_snake_case     =						"""cpu"""  # ensure determinism for the device-dependent torch.Generator
														_snake_case     =						self.dummy_cond_unet
														_snake_case     =						PNDMScheduler(skip_prk_steps=UpperCAmelCase						)
														_snake_case     =						self.dummy_vae
														_snake_case     =						self.dummy_text_encoder
														_snake_case     =						CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""						)

														# make sure here that pndm scheduler skips prk
														_snake_case     =						StableDiffusionPipeline(
														    unet=UpperCAmelCase						,						scheduler=UpperCAmelCase						,						vae=UpperCAmelCase						,						text_encoder=UpperCAmelCase						,						tokenizer=UpperCAmelCase						,						safety_checker=UpperCAmelCase						,						feature_extractor=self.dummy_extractor						,						)
														_snake_case     =						sd_pipe.to(UpperCAmelCase						)
														sd_pipe.set_progress_bar_config(disable=UpperCAmelCase						)

														_snake_case     =						"""A painting of a squirrel eating a burger"""
														_snake_case     =						torch.Generator(device=UpperCAmelCase						).manual_seed(0						)
														_snake_case     =						sd_pipe([prompt]						,						generator=UpperCAmelCase						,						guidance_scale=6.0						,						num_inference_steps=2						,						output_type="""np"""						)

														_snake_case     =						output.images

														_snake_case     =						torch.Generator(device=UpperCAmelCase						).manual_seed(0						)
														_snake_case     =						sd_pipe(
														    [prompt]						,						generator=UpperCAmelCase						,						guidance_scale=6.0						,						num_inference_steps=2						,						output_type="""np"""						,						return_dict=UpperCAmelCase						,						)[0]

														_snake_case     =						image[0, -3:, -3:, -1]
														_snake_case     =						image_from_tuple[0, -3:, -3:, -1]

														assert image.shape == (1, 64, 64, 3)
														_snake_case     =						np.array([0.5122, 0.5712, 0.4825, 0.5053, 0.5646, 0.4769, 0.5179, 0.4894, 0.4994]						)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2
														assert np.abs(image_from_tuple_slice.flatten() - expected_slice						).max() < 1e-2

							def  lowercase							(self						)  ->     Optional[int]:
														_snake_case     =						StableDiffusionPipeline.from_pretrained(
														    """hf-internal-testing/tiny-stable-diffusion-lms-pipe"""						,						safety_checker=UpperCAmelCase						)
														assert isinstance(UpperCAmelCase						,						UpperCAmelCase						)
														assert isinstance(pipe.scheduler						,						UpperCAmelCase						)
														assert pipe.safety_checker is None

														_snake_case     =						pipe("""example prompt"""						,						num_inference_steps=2						).images[0]
														assert image is not None

														# check that there's no error when saving a pipeline with one of the models being None
														with tempfile.TemporaryDirectory() as tmpdirname:
																					pipe.save_pretrained(UpperCAmelCase						)
																					_snake_case     =						StableDiffusionPipeline.from_pretrained(UpperCAmelCase						)

														# sanity check that the pipeline still works
														assert pipe.safety_checker is None
														_snake_case     =						pipe("""example prompt"""						,						num_inference_steps=2						).images[0]
														assert image is not None





							@unittest.skipIf(torch_device != """cuda"""						,						"""This test requires a GPU"""						)
							def  lowercase							(self						)  ->     Tuple:
														_snake_case     =						self.dummy_cond_unet
														_snake_case     =						PNDMScheduler(skip_prk_steps=UpperCAmelCase						)
														_snake_case     =						self.dummy_vae
														_snake_case     =						self.dummy_text_encoder
														_snake_case     =						CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""						)

														# put models in fp16
														_snake_case     =						unet.half()
														_snake_case     =						vae.half()
														_snake_case     =						bert.half()

														# make sure here that pndm scheduler skips prk
														_snake_case     =						StableDiffusionPipeline(
														    unet=UpperCAmelCase						,						scheduler=UpperCAmelCase						,						vae=UpperCAmelCase						,						text_encoder=UpperCAmelCase						,						tokenizer=UpperCAmelCase						,						safety_checker=UpperCAmelCase						,						feature_extractor=self.dummy_extractor						,						)
														_snake_case     =						sd_pipe.to(UpperCAmelCase						)
														sd_pipe.set_progress_bar_config(disable=UpperCAmelCase						)

														_snake_case     =						"""A painting of a squirrel eating a burger"""
														_snake_case     =						sd_pipe([prompt]						,						num_inference_steps=2						,						output_type="""np"""						).images

														assert image.shape == (1, 64, 64, 3)


@nightly
@require_torch_gpu
class     _lowerCAmelCase					(					unittest.TestCase   ):







							'''simple docstring'''

							def  lowercase							(self						)  ->     List[Any]:
														# clean up the VRAM after each test
														super().tearDown()
														gc.collect()
														torch.cuda.empty_cache()

							def  lowercase							(self						)  ->     str:
														_snake_case     =						"""cuda"""
														_snake_case     =						StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5"""						)
														_snake_case     =						pipe.to(UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=UpperCAmelCase						)

														_snake_case     =						"""a photo of a cat"""
														_snake_case     =						{
														    """editing_prompt""": ["""sunglasses"""],
														    """reverse_editing_direction""": [False],
														    """edit_warmup_steps""": 10,
														    """edit_guidance_scale""": 6,
														    """edit_threshold""": 0.95,
														    """edit_momentum_scale""": 0.5,
														    """edit_mom_beta""": 0.6,
														}

														_snake_case     =						3
														_snake_case     =						7

														# no sega enabled
														_snake_case     =						torch.Generator(UpperCAmelCase						)
														generator.manual_seed(UpperCAmelCase						)
														_snake_case     =						pipe(
														    [prompt]						,						generator=UpperCAmelCase						,						guidance_scale=UpperCAmelCase						,						num_inference_steps=50						,						output_type="""np"""						,						width=512						,						height=512						,						)

														_snake_case     =						output.images
														_snake_case     =						image[0, -3:, -3:, -1]
														_snake_case     =						[
														    0.3467_3113,
														    0.3849_2733,
														    0.3759_7352,
														    0.3408_6335,
														    0.3565_0748,
														    0.3557_9205,
														    0.338_4763,
														    0.3434_0236,
														    0.357_3271,
														]

														assert image.shape == (1, 512, 512, 3)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2

														# with sega enabled
														# generator = torch.manual_seed(seed)
														generator.manual_seed(UpperCAmelCase						)
														_snake_case     =						pipe(
														    [prompt]						,						generator=UpperCAmelCase						,						guidance_scale=UpperCAmelCase						,						num_inference_steps=50						,						output_type="""np"""						,						width=512						,						height=512						,						**UpperCAmelCase						,						)

														_snake_case     =						output.images
														_snake_case     =						image[0, -3:, -3:, -1]
														_snake_case     =						[
														    0.4188_7826,
														    0.3772_8766,
														    0.3013_8272,
														    0.4141_6335,
														    0.4166_4985,
														    0.3628_3392,
														    0.3619_1246,
														    0.4336_4465,
														    0.4300_1732,
														]

														assert image.shape == (1, 512, 512, 3)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2

							def  lowercase							(self						)  ->     Dict:
														_snake_case     =						"""cuda"""
														_snake_case     =						StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5"""						)
														_snake_case     =						pipe.to(UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=UpperCAmelCase						)

														_snake_case     =						"""an image of a crowded boulevard, realistic, 4k"""
														_snake_case     =						{
														    """editing_prompt""": """crowd, crowded, people""",
														    """reverse_editing_direction""": True,
														    """edit_warmup_steps""": 10,
														    """edit_guidance_scale""": 8.3,
														    """edit_threshold""": 0.9,
														    """edit_momentum_scale""": 0.5,
														    """edit_mom_beta""": 0.6,
														}

														_snake_case     =						9
														_snake_case     =						7

														# no sega enabled
														_snake_case     =						torch.Generator(UpperCAmelCase						)
														generator.manual_seed(UpperCAmelCase						)
														_snake_case     =						pipe(
														    [prompt]						,						generator=UpperCAmelCase						,						guidance_scale=UpperCAmelCase						,						num_inference_steps=50						,						output_type="""np"""						,						width=512						,						height=512						,						)

														_snake_case     =						output.images
														_snake_case     =						image[0, -3:, -3:, -1]
														_snake_case     =						[
														    0.4349_7998,
														    0.9181_4065,
														    0.754_0739,
														    0.5558_0205,
														    0.846_7265,
														    0.538_9691,
														    0.6257_4506,
														    0.5889_7763,
														    0.5092_6757,
														]

														assert image.shape == (1, 512, 512, 3)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2

														# with sega enabled
														# generator = torch.manual_seed(seed)
														generator.manual_seed(UpperCAmelCase						)
														_snake_case     =						pipe(
														    [prompt]						,						generator=UpperCAmelCase						,						guidance_scale=UpperCAmelCase						,						num_inference_steps=50						,						output_type="""np"""						,						width=512						,						height=512						,						**UpperCAmelCase						,						)

														_snake_case     =						output.images
														_snake_case     =						image[0, -3:, -3:, -1]
														_snake_case     =						[
														    0.308_9719,
														    0.3050_0144,
														    0.2901_6042,
														    0.3063_0964,
														    0.32_5687,
														    0.2941_9225,
														    0.290_8091,
														    0.2872_3598,
														    0.2769_6294,
														]

														assert image.shape == (1, 512, 512, 3)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2

							def  lowercase							(self						)  ->     Union[str, Any]:
														_snake_case     =						"""cuda"""
														_snake_case     =						StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5"""						)
														_snake_case     =						pipe.to(UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=UpperCAmelCase						)

														_snake_case     =						"""a castle next to a river"""
														_snake_case     =						{
														    """editing_prompt""": ["""boat on a river, boat""", """monet, impression, sunrise"""],
														    """reverse_editing_direction""": False,
														    """edit_warmup_steps""": [15, 18],
														    """edit_guidance_scale""": 6,
														    """edit_threshold""": [0.9, 0.8],
														    """edit_momentum_scale""": 0.5,
														    """edit_mom_beta""": 0.6,
														}

														_snake_case     =						48
														_snake_case     =						7

														# no sega enabled
														_snake_case     =						torch.Generator(UpperCAmelCase						)
														generator.manual_seed(UpperCAmelCase						)
														_snake_case     =						pipe(
														    [prompt]						,						generator=UpperCAmelCase						,						guidance_scale=UpperCAmelCase						,						num_inference_steps=50						,						output_type="""np"""						,						width=512						,						height=512						,						)

														_snake_case     =						output.images
														_snake_case     =						image[0, -3:, -3:, -1]
														_snake_case     =						[
														    0.7516_3555,
														    0.7603_7145,
														    0.6_1785,
														    0.918_9673,
														    0.862_7701,
														    0.8518_9694,
														    0.851_2813,
														    0.8701_2076,
														    0.831_2857,
														]

														assert image.shape == (1, 512, 512, 3)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2

														# with sega enabled
														# generator = torch.manual_seed(seed)
														generator.manual_seed(UpperCAmelCase						)
														_snake_case     =						pipe(
														    [prompt]						,						generator=UpperCAmelCase						,						guidance_scale=UpperCAmelCase						,						num_inference_steps=50						,						output_type="""np"""						,						width=512						,						height=512						,						**UpperCAmelCase						,						)

														_snake_case     =						output.images
														_snake_case     =						image[0, -3:, -3:, -1]
														_snake_case     =						[
														    0.7355_3365,
														    0.753_7271,
														    0.7434_1905,
														    0.6648_0356,
														    0.647_2925,
														    0.6303_9416,
														    0.6481_2905,
														    0.674_9717,
														    0.651_7102,
														]

														assert image.shape == (1, 512, 512, 3)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2





							def  lowercase							(self						)  ->     str:
														_snake_case     =						"""cuda"""
														_snake_case     =						StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5"""						,						torch_dtype=torch.floataa						)
														_snake_case     =						pipe.to(UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=UpperCAmelCase						)

														_snake_case     =						"""a photo of a cat"""
														_snake_case     =						{
														    """editing_prompt""": ["""sunglasses"""],
														    """reverse_editing_direction""": [False],
														    """edit_warmup_steps""": 10,
														    """edit_guidance_scale""": 6,
														    """edit_threshold""": 0.95,
														    """edit_momentum_scale""": 0.5,
														    """edit_mom_beta""": 0.6,
														}

														_snake_case     =						3
														_snake_case     =						7

														# no sega enabled
														_snake_case     =						torch.Generator(UpperCAmelCase						)
														generator.manual_seed(UpperCAmelCase						)
														_snake_case     =						pipe(
														    [prompt]						,						generator=UpperCAmelCase						,						guidance_scale=UpperCAmelCase						,						num_inference_steps=50						,						output_type="""np"""						,						width=512						,						height=512						,						)

														_snake_case     =						output.images
														_snake_case     =						image[0, -3:, -3:, -1]
														_snake_case     =						[
														    0.3488_7695,
														    0.387_6953,
														    0.375,
														    0.3442_3828,
														    0.358_1543,
														    0.3571_7773,
														    0.338_3789,
														    0.3457_0312,
														    0.35_9375,
														]

														assert image.shape == (1, 512, 512, 3)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2

														# with sega enabled
														# generator = torch.manual_seed(seed)
														generator.manual_seed(UpperCAmelCase						)
														_snake_case     =						pipe(
														    [prompt]						,						generator=UpperCAmelCase						,						guidance_scale=UpperCAmelCase						,						num_inference_steps=50						,						output_type="""np"""						,						width=512						,						height=512						,						**UpperCAmelCase						,						)

														_snake_case     =						output.images
														_snake_case     =						image[0, -3:, -3:, -1]
														_snake_case     =						[
														    0.4228_5156,
														    0.3691_4062,
														    0.2907_7148,
														    0.4204_1016,
														    0.4191_8945,
														    0.3549_8047,
														    0.361_8164,
														    0.442_3828,
														    0.4311_5234,
														]

														assert image.shape == (1, 512, 512, 3)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2

'''simple docstring'''
import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


__lowerCAmelCase  							= logging.get_logger(__name__)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def __init__(self						)  ->     Any:
														super().__init__()
														if version.parse(torch.__version__						) < version.parse("""1.12.0"""						):
																					raise ImportError(
																					    f"""You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use """
																					    """PytorchGELUTanh. Please upgrade torch."""						)





							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return nn.functional.gelu(UpperCAmelCase						,						approximate="""tanh"""						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi						) * (input + 0.04_4715 * torch.pow(UpperCAmelCase						,						3.0						))						))


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def __init__(self						,						UpperCAmelCase = False						)  ->     Optional[Any]:
														super().__init__()
														if use_gelu_python:
																					_snake_case     =						self._gelu_python
														else:
																					_snake_case     =						nn.functional.gelu

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0						)						))





							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return self.act(UpperCAmelCase						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return 0.5 * input * (1.0 + torch.tanh(input * 0.79_7884_5608 * (1.0 + 0.04_4715 * input * input)						))


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return input * torch.sigmoid(1.702 * input						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def __init__(self						,						UpperCAmelCase						,						UpperCAmelCase						)  ->     List[Any]:
														if min > max:
																					raise ValueError(f"""min should be < max (got min: {min}, max: {max})"""						)

														super().__init__()
														_snake_case     =						min
														_snake_case     =						max





							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return torch.clip(gelu(UpperCAmelCase						)						,						self.min						,						self.max						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def __init__(self						)  ->     Tuple:
														super().__init__()
														_snake_case     =						math.sqrt(2 / math.pi						)





							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_4715 * torch.pow(UpperCAmelCase						,						3						))						))


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return nn.functional.silu(UpperCAmelCase						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def __init__(self						)  ->     Any:
														super().__init__()
														if version.parse(torch.__version__						) < version.parse("""1.9.0"""						):
																					_snake_case     =						self._mish_python
														else:
																					_snake_case     =						nn.functional.mish

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return input * torch.tanh(nn.functional.softplus(UpperCAmelCase						)						)





							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return self.act(UpperCAmelCase						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return input


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase=0.70_7107						,						UpperCAmelCase=0.28_2095						)  ->     List[str]:
														_snake_case     =						(input - mu).div(sigma * math.sqrt(2.0						)						)
														return 0.5 * (1.0 + torch.erf(UpperCAmelCase						))


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Any:
														_snake_case     =						nn.functional.relu(UpperCAmelCase						)
														_snake_case     =						torch.square(UpperCAmelCase						)
														return squared


class     _lowerCAmelCase					(					__snake_case   ):







							'''simple docstring'''

							def __getitem__(self						,						UpperCAmelCase						)  ->     Tuple:
														_snake_case     =						super().__getitem__(UpperCAmelCase						)
														_snake_case,							_snake_case     =						content if isinstance(UpperCAmelCase						,						UpperCAmelCase						) else (content, {})
														return cls(**UpperCAmelCase						)
__lowerCAmelCase  							= {
    'gelu': GELUActivation,
    'gelu_10': (ClippedGELUActivation, {'min': -10, 'max': 10}),
    'gelu_fast': FastGELUActivation,
    'gelu_new': NewGELUActivation,
    'gelu_python': (GELUActivation, {'use_gelu_python': True}),
    'gelu_pytorch_tanh': PytorchGELUTanh,
    'gelu_accurate': AccurateGELUActivation,
    'laplace': LaplaceActivation,
    'linear': LinearActivation,
    'mish': MishActivation,
    'quick_gelu': QuickGELUActivation,
    'relu': nn.ReLU,
    'relu2': ReLUSquaredActivation,
    'relu6': nn.ReLUa,
    'sigmoid': nn.Sigmoid,
    'silu': SiLUActivation,
    'swish': SiLUActivation,
    'tanh': nn.Tanh,
}
__lowerCAmelCase  							= ClassInstantier(ACTaCLS)






def       __SCREAMING_SNAKE_CASE   (						_SCREAMING_SNAKE_CASE   ):
							if activation_string in ACTaFN:
														return ACTaFN[activation_string]
							else:
														raise KeyError(f"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()   )}"""   )


# For backwards compatibility with: from activations import gelu_python
__lowerCAmelCase  							= get_activation('gelu_python')
__lowerCAmelCase  							= get_activation('gelu_new')
__lowerCAmelCase  							= get_activation('gelu')
__lowerCAmelCase  							= get_activation('gelu_fast')
__lowerCAmelCase  							= get_activation('quick_gelu')
__lowerCAmelCase  							= get_activation('silu')
__lowerCAmelCase  							= get_activation('mish')

__lowerCAmelCase  							= get_activation('linear')