


import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


UpperCAmelCase      :						Union[str, Any]    =		logging.get_logger(__name__)







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def __init__(   self  )     ->   str:







     '''simple docstring'''







     super().__init__()
     if version.parse(torch.__version__  ) < version.parse("""1.12.0"""  ):
        raise ImportError(
            F'You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '
            """PytorchGELUTanh. Please upgrade torch."""  )







  def 				__A      (   self		,			A  )     ->   Optional[Any]:







     '''simple docstring'''







     return nn.functional.gelu(_lowerCamelCase		,			approximate="""tanh"""  )







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def 				__A      (   self		,			A  )     ->   List[Any]:







     '''simple docstring'''







     return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi  ) * (input + 0.044715 * torch.pow(_lowerCamelCase		,			3.0  ))  ))







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def __init__(   self		,			A = False  )     ->   str:







     '''simple docstring'''







     super().__init__()
     if use_gelu_python:
        lowerCamelCase								=     self._gelu_python
     else:
        lowerCamelCase								=     nn.functional.gelu







  def 				__A      (   self		,			A  )     ->   Any:







     '''simple docstring'''







     return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0  )  ))







  def 				__A      (   self		,			A  )     ->   Dict:







     '''simple docstring'''







     return self.act(_lowerCamelCase  )







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def 				__A      (   self		,			A  )     ->   Any:







     '''simple docstring'''







     return 0.5 * input * (1.0 + torch.tanh(input * 0.7978845608 * (1.0 + 0.044715 * input * input)  ))







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def 				__A      (   self		,			A  )     ->   List[str]:







     '''simple docstring'''







     return input * torch.sigmoid(1.702 * input  )







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def __init__(   self		,			A		,			A  )     ->   Optional[int]:







     '''simple docstring'''







     if min > max:
        raise ValueError(F'min should be < max (got min: {min}, max: {max})'  )

     super().__init__()
     lowerCamelCase								=     min
     lowerCamelCase								=     max







  def 				__A      (   self		,			A  )     ->   int:







     '''simple docstring'''







     return torch.clip(gelu(_lowerCamelCase  )		,			self.min		,			self.max  )







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def __init__(   self  )     ->   Union[str, Any]:







     '''simple docstring'''







     super().__init__()
     lowerCamelCase								=     math.sqrt(2 / math.pi  )







  def 				__A      (   self		,			A  )     ->   Optional[int]:







     '''simple docstring'''







     return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044715 * torch.pow(_lowerCamelCase		,			3  ))  ))







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def 				__A      (   self		,			A  )     ->   List[Any]:







     '''simple docstring'''







     return nn.functional.silu(_lowerCamelCase  )







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def __init__(   self  )     ->   List[str]:







     '''simple docstring'''







     super().__init__()
     if version.parse(torch.__version__  ) < version.parse("""1.9.0"""  ):
        lowerCamelCase								=     self._mish_python
     else:
        lowerCamelCase								=     nn.functional.mish







  def 				__A      (   self		,			A  )     ->   str:







     '''simple docstring'''







     return input * torch.tanh(nn.functional.softplus(_lowerCamelCase  )  )







  def 				__A      (   self		,			A  )     ->   List[str]:







     '''simple docstring'''







     return self.act(_lowerCamelCase  )







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def 				__A      (   self		,			A  )     ->   str:







     '''simple docstring'''







     return input







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def 				__A      (   self		,			A		,			A=0.707107		,			A=0.282095  )     ->   Optional[Any]:







     '''simple docstring'''







     lowerCamelCase								=     (input - mu).div(sigma * math.sqrt(2.0  )  )
     return 0.5 * (1.0 + torch.erf(_lowerCamelCase  ))







class 					__lowercase					(       nn.Module     ):



  """simple docstring"""

  def 				__A      (   self		,			A  )     ->   List[str]:







     '''simple docstring'''







     lowerCamelCase								=     nn.functional.relu(_lowerCamelCase  )
     lowerCamelCase								=     torch.square(_lowerCamelCase  )
     return squared







class 					__lowercase					(       lowerCAmelCase__     ):



  """simple docstring"""

  def __getitem__(   self		,			A  )     ->   Optional[Any]:







     '''simple docstring'''







     lowerCamelCase								=     super().__getitem__(_lowerCamelCase  )
     lowerCamelCase								=     content if isinstance(_lowerCamelCase		,			_lowerCamelCase  ) else (content, {})
     return cls(**_lowerCamelCase  )






UpperCAmelCase      :						Optional[Any]    =		{
    "gelu": GELUActivation,
    "gelu_10": (ClippedGELUActivation, {"min": -10, "max": 10}),
    "gelu_fast": FastGELUActivation,
    "gelu_new": NewGELUActivation,
    "gelu_python": (GELUActivation, {"use_gelu_python": True}),
    "gelu_pytorch_tanh": PytorchGELUTanh,
    "gelu_accurate": AccurateGELUActivation,
    "laplace": LaplaceActivation,
    "linear": LinearActivation,
    "mish": MishActivation,
    "quick_gelu": QuickGELUActivation,
    "relu": nn.ReLU,
    "relu2": ReLUSquaredActivation,
    "relu6": nn.ReLUa,
    "sigmoid": nn.Sigmoid,
    "silu": SiLUActivation,
    "swish": SiLUActivation,
    "tanh": nn.Tanh,
}
UpperCAmelCase      :						List[Any]    =		ClassInstantier(ACTaCLS)






def   __lowerCamelCase	(       lowerCamelCase__		:	str			):





   '''simple docstring'''




   if activation_string in ACTaFN:
      return ACTaFN[activation_string]
   else:
      raise KeyError(f'function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()			)}'			)


# For backwards compatibility with: from activations import gelu_python
UpperCAmelCase      :						str    =		get_activation("gelu_python")
UpperCAmelCase      :						List[str]    =		get_activation("gelu_new")
UpperCAmelCase      :						List[Any]    =		get_activation("gelu")
UpperCAmelCase      :						Tuple    =		get_activation("gelu_fast")
UpperCAmelCase      :						int    =		get_activation("quick_gelu")
UpperCAmelCase      :						List[Any]    =		get_activation("silu")
UpperCAmelCase      :						Optional[int]    =		get_activation("mish")



UpperCAmelCase      :						Optional[int]    =		get_activation("linear")





def   __lowerCamelCase	(       lowerCamelCase__		:	int = 100     ,       lowerCamelCase__		:	int = 100			):





			'''simple docstring'''




			return max(
			    sum(int(lowerCamelCase__			) for x in str(base**power			)			)
			    for base in range(lowerCamelCase__			)
			    for power in range(lowerCamelCase__			)			)


# Tests
if __name__ == "__main__":
		import doctest

		doctest.testmod()

