






'''simple docstring'''







import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


__A   :							Any     	=						logging.get_logger(__name__)






class        a__     (      nn.Module    ):
     '''simple docstring'''







     def __init__(self     :						Dict       )	->							List[str]:
           super().__init__()
           if version.parse(torch.__version__       ) < version.parse("1.12.0"       ):
                 raise ImportError(
                     f"You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use "
                     "PytorchGELUTanh. Please upgrade torch."       )






     def      _UpperCamelCase (self     :						str					,					lowercase__     :						Tensor       )	->							Tensor:
           return nn.functional.gelu(lowercase__					,					approximate="tanh"       )







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def      _UpperCamelCase (self     :						str					,					lowercase__     :						Tensor       )	->							Tensor:
           return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi       ) * (input + 0.04_47_15 * torch.pow(lowercase__					,					3.0       ))       ))







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def __init__(self     :						Dict					,					lowercase__     :						bool = False       )	->							str:
           super().__init__()
           if use_gelu_python:
                 __A		 =   self._gelu_python
           else:
                 __A		 =   nn.functional.gelu






     def      _UpperCamelCase (self     :						List[Any]					,					lowercase__     :						Tensor       )	->							Tensor:
           return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0       )       ))






     def      _UpperCamelCase (self     :						Union[str, Any]					,					lowercase__     :						Tensor       )	->							Tensor:
           return self.act(lowercase__       )







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def      _UpperCamelCase (self     :						List[str]					,					lowercase__     :						Tensor       )	->							Tensor:
           return 0.5 * input * (1.0 + torch.tanh(input * 0.79_78_84_56_08 * (1.0 + 0.04_47_15 * input * input)       ))







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def      _UpperCamelCase (self     :						Union[str, Any]					,					lowercase__     :						Tensor       )	->							Tensor:
           return input * torch.sigmoid(1.7_02 * input       )







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def __init__(self     :						Optional[Any]					,					lowercase__     :						float					,					lowercase__     :						float       )	->							Any:
           if min > max:
                 raise ValueError(f"min should be < max (got min: {min}, max: {max})"       )

           super().__init__()
           __A		 =   min
           __A		 =   max






     def      _UpperCamelCase (self     :						Tuple					,					lowercase__     :						Tensor       )	->							Tensor:
           return torch.clip(gelu(lowercase__       )					,					self.min					,					self.max       )







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def __init__(self     :						Tuple       )	->							Union[str, Any]:
           super().__init__()
           __A		 =   math.sqrt(2 / math.pi       )






     def      _UpperCamelCase (self     :						Tuple					,					lowercase__     :						Tensor       )	->							Tensor:
           return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_47_15 * torch.pow(lowercase__					,					3       ))       ))







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def      _UpperCamelCase (self     :						Union[str, Any]					,					lowercase__     :						Tensor       )	->							Tensor:
           return nn.functional.silu(lowercase__       )







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def __init__(self     :						Optional[Any]       )	->							Dict:
           super().__init__()
           if version.parse(torch.__version__       ) < version.parse("1.9.0"       ):
                 __A		 =   self._mish_python
           else:
                 __A		 =   nn.functional.mish






     def      _UpperCamelCase (self     :						str					,					lowercase__     :						Tensor       )	->							Tensor:
           return input * torch.tanh(nn.functional.softplus(lowercase__       )       )






     def      _UpperCamelCase (self     :						List[Any]					,					lowercase__     :						Tensor       )	->							Tensor:
           return self.act(lowercase__       )







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def      _UpperCamelCase (self     :						Tuple					,					lowercase__     :						Tensor       )	->							Tensor:
           return input







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def      _UpperCamelCase (self     :						int					,					lowercase__     :						str					,					lowercase__     :						str=0.70_71_07					,					lowercase__     :						Union[str, Any]=0.28_20_95       )	->							Tuple:
           __A		 =   (input - mu).div(sigma * math.sqrt(2.0       )       )
           return 0.5 * (1.0 + torch.erf(lowercase__       ))







class        a__     (      nn.Module    ):
     '''simple docstring'''







     def      _UpperCamelCase (self     :						Optional[Any]					,					lowercase__     :						Optional[int]       )	->							str:
           __A		 =   nn.functional.relu(lowercase__       )
           __A		 =   torch.square(lowercase__       )
           return squared







class        a__     (      lowerCamelCase    ):
     '''simple docstring'''







     def __getitem__(self     :						List[Any]					,					lowercase__     :						Optional[Any]       )	->							Optional[Any]:
           __A		 =   super().__getitem__(lowercase__       )
           __A ,					__A		 =   content if isinstance(lowercase__					,					lowercase__       ) else (content, {})
           return cls(**lowercase__       )







__A   :							List[str]     	=						{
    'gelu': GELUActivation,
    'gelu_10': (ClippedGELUActivation, {'min': -1_0, 'max': 1_0}),
    'gelu_fast': FastGELUActivation,
    'gelu_new': NewGELUActivation,
    'gelu_python': (GELUActivation, {'use_gelu_python': True}),
    'gelu_pytorch_tanh': PytorchGELUTanh,
    'gelu_accurate': AccurateGELUActivation,
    'laplace': LaplaceActivation,
    'linear': LinearActivation,
    'mish': MishActivation,
    'quick_gelu': QuickGELUActivation,
    'relu': nn.ReLU,
    'relu2': ReLUSquaredActivation,
    'relu6': nn.ReLUa,
    'sigmoid': nn.Sigmoid,
    'silu': SiLUActivation,
    'swish': SiLUActivation,
    'tanh': nn.Tanh,
}
__A   :							str     	=						ClassInstantier(ACTaCLS)


def     _UpperCamelCase				(  snake_case__				):
      if activation_string in ACTaFN:
            return ACTaFN[activation_string]
      else:
            raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()				)}"				)


# For backwards compatibility with: from activations import gelu_python
__A   :							List[Any]     	=						get_activation('gelu_python')
__A   :							List[Any]     	=						get_activation('gelu_new')
__A   :							Tuple     	=						get_activation('gelu')
__A   :							Optional[Any]     	=						get_activation('gelu_fast')
__A   :							Optional[int]     	=						get_activation('quick_gelu')
__A   :							int     	=						get_activation('silu')
__A   :							Any     	=						get_activation('mish')





__A   :							List[str]     	=						get_activation('linear')










'''simple docstring'''







import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_multi_gpu,
    require_single_gpu,
    test_sync,
)
from accelerate.utils import patch_environment






class        a__     (      unittest.TestCase    ):
     '''simple docstring'''







     def      _UpperCamelCase (self     :						int       )	->							Optional[int]:
           __A		 =   inspect.getfile(accelerate.test_utils       )
           __A		 =   os.path.sep.join(mod_file.split(os.path.sep       )[:-1] + ["scripts", "test_sync.py"]       )






     @require_cpu
     def      _UpperCamelCase (self     :						str       )	->							Optional[Any]:
           debug_launcher(test_sync.main					,					num_processes=1       )






     @require_cpu
     def      _UpperCamelCase (self     :						Union[str, Any]       )	->							Optional[Any]:
           debug_launcher(test_sync.main       )






     @require_single_gpu
     def      _UpperCamelCase (self     :						Union[str, Any]       )	->							List[str]:
           test_sync.main()






     @require_multi_gpu
     def      _UpperCamelCase (self     :						Optional[int]       )	->							List[str]:
           print(f"Found {torch.cuda.device_count()} devices."       )
           __A		 =   ["torchrun", f"--nproc_per_node={torch.cuda.device_count()}", self.test_file_path]
           with patch_environment(omp_num_threads=1       ):
                 execute_subprocess_async(lowercase__					,					env=os.environ.copy()       )


