import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


__snake_case         =     logging.get_logger(__name__)







class       __snake_case (  nn.Module ):
 def __init__(	self    )						->				List[Any]:
       '''simple docstring'''





       super().__init__()
       if version.parse(torch.__version__    ) < version.parse('''1.12.0'''    ):
             raise ImportError(
                 f'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
                 '''PytorchGELUTanh. Please upgrade torch.'''    )






 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return nn.functional.gelu(snake_case__		, approximate='''tanh'''    )







class       __snake_case (  nn.Module ):
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi    ) * (input + 0.04_4715 * torch.pow(snake_case__		, 3.0    ))    ))







class       __snake_case (  nn.Module ):
 def __init__(	self		, snake_case__ = False    )						->				Optional[int]:
       '''simple docstring'''





       super().__init__()
       if use_gelu_python:
             UpperCAmelCase     :			Optional[int]     	=self._gelu_python
       else:
             UpperCAmelCase     :			Dict     	=nn.functional.gelu
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0    )    ))






 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return self.act(snake_case__    )







class       __snake_case (  nn.Module ):
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return 0.5 * input * (1.0 + torch.tanh(input * 0.79_7884_5608 * (1.0 + 0.04_4715 * input * input)    ))







class       __snake_case (  nn.Module ):
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return input * torch.sigmoid(1.702 * input    )







class       __snake_case (  nn.Module ):
 def __init__(	self		, snake_case__		, snake_case__    )						->				Optional[int]:
       '''simple docstring'''





       if min > max:
             raise ValueError(f'''min should be < max (got min: {min}, max: {max})'''    )

       super().__init__()
       UpperCAmelCase     :			str     	=min
       UpperCAmelCase     :			Union[str, Any]     	=max






 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return torch.clip(gelu(snake_case__    )		, self.min		, self.max    )







class       __snake_case (  nn.Module ):
 def __init__(	self    )						->				Dict:
       '''simple docstring'''





       super().__init__()
       UpperCAmelCase     :			List[Any]     	=math.sqrt(2 / math.pi    )






 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_4715 * torch.pow(snake_case__		, 3    ))    ))







class       __snake_case (  nn.Module ):
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return nn.functional.silu(snake_case__    )







class       __snake_case (  nn.Module ):
 def __init__(	self    )						->				Optional[int]:
       '''simple docstring'''





       super().__init__()
       if version.parse(torch.__version__    ) < version.parse('''1.9.0'''    ):
             UpperCAmelCase     :			List[Any]     	=self._mish_python
       else:
             UpperCAmelCase     :			List[Any]     	=nn.functional.mish
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return input * torch.tanh(nn.functional.softplus(snake_case__    )    )






 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return self.act(snake_case__    )







class       __snake_case (  nn.Module ):
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tensor:
       '''simple docstring'''





       return input







class       __snake_case (  nn.Module ):
 def 						UpperCAmelCase__							(	self		, snake_case__		, snake_case__=0.70_7107		, snake_case__=0.28_2095    )						->				Dict:
       '''simple docstring'''





       UpperCAmelCase     :			Tuple     	=(input - mu).div(sigma * math.sqrt(2.0    )    )
       return 0.5 * (1.0 + torch.erf(snake_case__    ))







class       __snake_case (  nn.Module ):
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Dict:
       '''simple docstring'''





       UpperCAmelCase     :			Optional[int]     	=nn.functional.relu(snake_case__    )
       UpperCAmelCase     :			Union[str, Any]     	=torch.square(snake_case__    )
       return squared







class       __snake_case (  lowerCamelCase__ ):
 def __getitem__(	self		, snake_case__    )						->				Optional[Any]:
       '''simple docstring'''





       UpperCAmelCase     :			Optional[int]     	=super().__getitem__(snake_case__    )
       UpperCAmelCase	,    UpperCAmelCase     :			List[str]     	=content if isinstance(snake_case__		, snake_case__    ) else (content, {})
       return cls(**snake_case__    )



__snake_case         =     {
    '''gelu''': GELUActivation,
    '''gelu_10''': (ClippedGELUActivation, {'''min''': -10, '''max''': 10}),
    '''gelu_fast''': FastGELUActivation,
    '''gelu_new''': NewGELUActivation,
    '''gelu_python''': (GELUActivation, {'''use_gelu_python''': True}),
    '''gelu_pytorch_tanh''': PytorchGELUTanh,
    '''gelu_accurate''': AccurateGELUActivation,
    '''laplace''': LaplaceActivation,
    '''linear''': LinearActivation,
    '''mish''': MishActivation,
    '''quick_gelu''': QuickGELUActivation,
    '''relu''': nn.ReLU,
    '''relu2''': ReLUSquaredActivation,
    '''relu6''': nn.ReLUa,
    '''sigmoid''': nn.Sigmoid,
    '''silu''': SiLUActivation,
    '''swish''': SiLUActivation,
    '''tanh''': nn.Tanh,
}
__snake_case         =     ClassInstantier(ACTaCLS)





def 	lowerCAmelCase_			(					__lowerCAmelCase    )->	Tuple:






      '''simple docstring'''





      if activation_string in ACTaFN:
            return ACTaFN[activation_string]
      else:
            raise KeyError(f'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()    )}'''    )


# For backwards compatibility with: from activations import gelu_python
__snake_case         =     get_activation('''gelu_python''')
__snake_case         =     get_activation('''gelu_new''')
__snake_case         =     get_activation('''gelu''')
__snake_case         =     get_activation('''gelu_fast''')
__snake_case         =     get_activation('''quick_gelu''')
__snake_case         =     get_activation('''silu''')
__snake_case         =     get_activation('''mish''')


__snake_case         =     get_activation('''linear''')

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


__snake_case         =     {'''configuration_gptj''': ['''GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''GPTJConfig''', '''GPTJOnnxConfig''']}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __snake_case         =     [
         '''GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''GPTJForCausalLM''',
         '''GPTJForQuestionAnswering''',
         '''GPTJForSequenceClassification''',
         '''GPTJModel''',
         '''GPTJPreTrainedModel''',
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __snake_case         =     [
         '''TFGPTJForCausalLM''',
         '''TFGPTJForQuestionAnswering''',
         '''TFGPTJForSequenceClassification''',
         '''TFGPTJModel''',
         '''TFGPTJPreTrainedModel''',
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __snake_case         =     [
         '''FlaxGPTJForCausalLM''',
         '''FlaxGPTJModel''',
         '''FlaxGPTJPreTrainedModel''',
     ]


if TYPE_CHECKING:
     from .configuration_gptj import GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTJConfig, GPTJOnnxConfig

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_gptj import (
              GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST,
              GPTJForCausalLM,
              GPTJForQuestionAnswering,
              GPTJForSequenceClassification,
              GPTJModel,
              GPTJPreTrainedModel,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_gptj import (
              TFGPTJForCausalLM,
              TFGPTJForQuestionAnswering,
              TFGPTJForSequenceClassification,
              TFGPTJModel,
              TFGPTJPreTrainedModel,
          )

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_gptj import FlaxGPTJForCausalLM, FlaxGPTJModel, FlaxGPTJPreTrainedModel

else:
     import sys

     __snake_case         =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
