




import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


UpperCAmelCase__    :							List[Any]       		=     logging.get_logger(__name__)

class   UpperCAmelCase    (      nn.Module ):






   def __init__(						self :							str  )->      Union[str, Any]:
     super().__init__()
     if version.parse(torch.__version__  ) < version.parse('1.12.0'  ):
       raise ImportError(
           f'You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '
           'PytorchGELUTanh. Please upgrade torch.'  )






   def 			_a    (						self :							List[str]    ,       UpperCamelCase_ :							Tensor  )->      Union[str, Any]:
     return nn.functional.gelu(UpperCAmelCase_    ,       approximate='tanh'  )

class   UpperCAmelCase    (      nn.Module ):






   def 			_a    (						self :							Any    ,       UpperCamelCase_ :							Tensor  )->      Union[str, Any]:
     return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi  ) * (input + 0.044_715 * torch.pow(UpperCAmelCase_    ,       3.0  ))  ))

class   UpperCAmelCase    (      nn.Module ):






   def __init__(						self :							List[str]    ,       UpperCamelCase_ :							bool = False  )->      Tuple:
     super().__init__()
     if use_gelu_python:
       _lowercase:			List[Any]      =     self._gelu_python
     else:
       _lowercase:			int      =     nn.functional.gelu






   def 			_a    (						self :							Tuple    ,       UpperCamelCase_ :							Tensor  )->      Optional[Any]:
     return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0  )  ))






   def 			_a    (						self :							int    ,       UpperCamelCase_ :							Tensor  )->      Optional[int]:
     return self.act(UpperCAmelCase_  )

class   UpperCAmelCase    (      nn.Module ):






   def 			_a    (						self :							Any    ,       UpperCamelCase_ :							Tensor  )->      List[Any]:
     return 0.5 * input * (1.0 + torch.tanh(input * 0.7_978_845_608 * (1.0 + 0.044_715 * input * input)  ))

class   UpperCAmelCase    (      nn.Module ):






   def 			_a    (						self :							List[str]    ,       UpperCamelCase_ :							Tensor  )->      Optional[Any]:
     return input * torch.sigmoid(1.702 * input  )

class   UpperCAmelCase    (      nn.Module ):






   def __init__(						self :							Optional[Any]    ,       UpperCamelCase_ :							float    ,       UpperCamelCase_ :							float  )->      Union[str, Any]:







     if min > max:
       raise ValueError(f'min should be < max (got min: {min}, max: {max})'  )

     super().__init__()
     _lowercase:			Tuple      =     min
     _lowercase:			Optional[Any]      =     max






   def 			_a    (						self :							List[Any]    ,       UpperCamelCase_ :							Tensor  )->      int:
     return torch.clip(gelu(UpperCAmelCase_  )    ,       self.min    ,       self.max  )

class   UpperCAmelCase    (      nn.Module ):






   def __init__(						self :							Dict  )->      Dict:
     super().__init__()
     _lowercase:			int      =     math.sqrt(2 / math.pi  )






   def 			_a    (						self :							List[Any]    ,       UpperCamelCase_ :							Tensor  )->      Tuple:
     return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044_715 * torch.pow(UpperCAmelCase_    ,       3  ))  ))

class   UpperCAmelCase    (      nn.Module ):






   def 			_a    (						self :							Optional[Any]    ,       UpperCamelCase_ :							Tensor  )->      str:
     return nn.functional.silu(UpperCAmelCase_  )

class   UpperCAmelCase    (      nn.Module ):






   def __init__(						self :							int  )->      List[str]:
     super().__init__()
     if version.parse(torch.__version__  ) < version.parse('1.9.0'  ):
       _lowercase:			Any      =     self._mish_python
     else:
       _lowercase:			Optional[Any]      =     nn.functional.mish






   def 			_a    (						self :							Optional[Any]    ,       UpperCamelCase_ :							Tensor  )->      List[Any]:
     return input * torch.tanh(nn.functional.softplus(UpperCAmelCase_  )  )






   def 			_a    (						self :							List[Any]    ,       UpperCamelCase_ :							Tensor  )->      Union[str, Any]:
     return self.act(UpperCAmelCase_  )

class   UpperCAmelCase    (      nn.Module ):






   def 			_a    (						self :							Optional[int]    ,       UpperCamelCase_ :							Tensor  )->      int:
     return input

class   UpperCAmelCase    (      nn.Module ):






   def 			_a    (						self :							Optional[int]    ,       UpperCamelCase_ :							List[str]    ,       UpperCamelCase_ :							Any=0.707_107    ,       UpperCamelCase_ :							Optional[int]=0.282_095  )->      List[Any]:
     _lowercase:			Optional[int]      =     (input - mu).div(sigma * math.sqrt(2.0  )  )
     return 0.5 * (1.0 + torch.erf(UpperCAmelCase_  ))

class   UpperCAmelCase    (      nn.Module ):






   def 			_a    (						self :							Union[str, Any]    ,       UpperCamelCase_ :							Union[str, Any]  )->      Any:
     _lowercase:			str      =     nn.functional.relu(UpperCAmelCase_  )
     _lowercase:			Union[str, Any]      =     torch.square(UpperCAmelCase_  )
     return squared

class   UpperCAmelCase    (      a_ ):






   def __getitem__(						self :							List[str]    ,       UpperCamelCase_ :							List[str]  )->      Union[str, Any]:
     _lowercase:			List[Any]      =     super().__getitem__(UpperCAmelCase_  )
     _lowercase:			Union[str, Any]      =     content if isinstance(UpperCAmelCase_    ,       UpperCAmelCase_  ) else (content, {})
     return cls(**UpperCAmelCase_  )
UpperCAmelCase__    :							Union[str, Any]       		=     {
    """gelu""": GELUActivation,
    """gelu_10""": (ClippedGELUActivation, {"""min""": -10, """max""": 10}),
    """gelu_fast""": FastGELUActivation,
    """gelu_new""": NewGELUActivation,
    """gelu_python""": (GELUActivation, {"""use_gelu_python""": True}),
    """gelu_pytorch_tanh""": PytorchGELUTanh,
    """gelu_accurate""": AccurateGELUActivation,
    """laplace""": LaplaceActivation,
    """linear""": LinearActivation,
    """mish""": MishActivation,
    """quick_gelu""": QuickGELUActivation,
    """relu""": nn.ReLU,
    """relu2""": ReLUSquaredActivation,
    """relu6""": nn.ReLUa,
    """sigmoid""": nn.Sigmoid,
    """silu""": SiLUActivation,
    """swish""": SiLUActivation,
    """tanh""": nn.Tanh,
}
UpperCAmelCase__    :							Optional[int]       		=     ClassInstantier(ACTaCLS)


def 		_lowerCamelCase					(       UpperCamelCase				)      ->		Tuple:





  """simple docstring"""





  if activation_string in ACTaFN:
    return ACTaFN[activation_string]
  else:
    raise KeyError(F'function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()				)}'				)


# For backwards compatibility with: from activations import gelu_python
UpperCAmelCase__    :							str       		=     get_activation('gelu_python')
UpperCAmelCase__    :							Tuple       		=     get_activation('gelu_new')
UpperCAmelCase__    :							List[Any]       		=     get_activation('gelu')
UpperCAmelCase__    :							str       		=     get_activation('gelu_fast')
UpperCAmelCase__    :							Optional[Any]       		=     get_activation('quick_gelu')
UpperCAmelCase__    :							Union[str, Any]       		=     get_activation('silu')
UpperCAmelCase__    :							str       		=     get_activation('mish')





UpperCAmelCase__    :							Tuple       		=     get_activation('linear')

'''simple docstring'''





import glob
import os

from get_test_info import get_tester_classes


if __name__ == "__main__":
      UpperCamelCase       :     List[Any]			=    []

      UpperCamelCase       :     Tuple			=    os.path.join("""tests""", """models""", """**""", """test_modeling_*.py""")
      UpperCamelCase       :     List[str]			=    glob.glob(pattern)
      # TODO: deal with TF/Flax too
      UpperCamelCase       :     List[Any]			=    [
          x for x in test_files if not (x.startswith("""test_modeling_tf_""") or x.startswith("""test_modeling_flax_"""))
      ]

      for test_file in test_files:
            UpperCamelCase       :     List[str]			=    get_tester_classes(test_file)
            for tester_class in tester_classes:
                  # A few tester classes don't have `parent` parameter in `__init__`.
                  # TODO: deal this better
                  try:
                        UpperCamelCase       :     int			=    tester_class(parent=None)
                  except Exception:
                        continue
                  if hasattr(tester, """get_config"""):
                        UpperCamelCase       :     Union[str, Any]			=    tester.get_config()
                        for k, v in config.to_dict().items():
                              if isinstance(v, int):
                                    UpperCamelCase       :     List[Any]			=    None
                                    if k in ["vocab_size"]:
                                          UpperCamelCase       :     List[Any]			=    100
                                    elif k in ["max_position_embeddings"]:
                                          UpperCamelCase       :     Tuple			=    128
                                    elif k in ["hidden_size", "d_model"]:
                                          UpperCamelCase       :     Any			=    40
                                    elif k == ["num_layers", "num_hidden_layers", "num_encoder_layers", "num_decoder_layers"]:
                                          UpperCamelCase       :     Tuple			=    5
                                    if target is not None and v > target:
                                          failures.append(
                                              f'''{tester_class.__name__} will produce a `config` of type `{config.__class__.__name__}`'''
                                              f''' with config["{k}"] = {v} which is too large for testing! Set its value to be smaller'''
                                              f''' than {target}.'''
                                          )

      if len(failures) > 0:
            raise Exception(f'''There were {len(failures)} failures:\n''' + """\n""".join(failures))

