


'''simple docstring'''

import unittest

import numpy as np

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
 import tensorflow as tf

 from transformers.activations_tf import get_tf_activation
@require_tf
class 						__lowerCAmelCase		(unittest.TestCase  ):





      '''simple docstring'''






      def    UpperCamelCase__	(self					:					List[Any]				):

             '''simple docstring'''







             lowercase__       =				tf.constant([-100, -1.0, -0.1, 0, 0.1, 1.0, 1_00.0]				)
             lowercase__       =				get_tf_activation('''gelu'''				)
             lowercase__       =				get_tf_activation('''gelu_10'''				)

             lowercase__       =				gelu(UpperCamelCase				)
             lowercase__       =				geluaa(UpperCamelCase				)

             lowercase__       =				tf.where(y_gelu_aa < 10.0     ,		1.0     ,		0.0				)

             self.assertEqual(tf.math.reduce_max(UpperCamelCase				).numpy().item()     ,		10.0				)
             self.assertTrue(np.allclose(y_gelu * clipped_mask     ,		y_gelu_aa * clipped_mask				)				)






      def    UpperCamelCase__	(self					:					List[str]				):

             '''simple docstring'''







             get_tf_activation('''gelu'''				)
             get_tf_activation('''gelu_10'''				)
             get_tf_activation('''gelu_fast'''				)
             get_tf_activation('''gelu_new'''				)
             get_tf_activation('''glu'''				)
             get_tf_activation('''mish'''				)
             get_tf_activation('''quick_gelu'''				)
             get_tf_activation('''relu'''				)
             get_tf_activation('''sigmoid'''				)
             get_tf_activation('''silu'''				)
             get_tf_activation('''swish'''				)
             get_tf_activation('''tanh'''				)
             with self.assertRaises(UpperCamelCase				):
                    get_tf_activation('''bogus'''				)
             with self.assertRaises(UpperCamelCase				):
                    get_tf_activation(UpperCamelCase				)








'''simple docstring'''

import torch

from accelerate import Accelerator, DistributedType
class 						__lowerCAmelCase		:





      '''simple docstring'''






      def __enter__(self					:					List[Any]				):

             '''simple docstring'''







             if self.enabled:
                    lowercase__       =				self.model.no_sync()
                    self.model_sync_obj.__enter__()

             return self



      def __exit__(self					:					List[str]     ,		UpperCamelCase					:					List[Any]     ,		UpperCamelCase					:					Optional[int]     ,		UpperCamelCase					:					Optional[int]				):

             '''simple docstring'''







             if self.enabled:
                    # Average all models on exit
                    self._sync_and_avg_model_params()
                    self.model_sync_obj.__exit__(UpperCamelCase     ,		UpperCamelCase     ,		UpperCamelCase				)



      def __init__(self					:					Dict     ,		UpperCamelCase					:					Accelerator     ,		UpperCamelCase					:					torch.nn.Module     ,		UpperCamelCase					:					int     ,		UpperCamelCase					:					bool = True				):

             '''simple docstring'''







             if accelerator.distributed_type not in [
                 DistributedType.NO,
                 DistributedType.MULTI_CPU,
                 DistributedType.MULTI_GPU,
             ]:
                    raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''				)
             lowercase__       =				enabled and accelerator.distributed_type != DistributedType.NO
             lowercase__       =				0
             if self.enabled:
                    lowercase__       =				accelerator
                    lowercase__       =				model
                    lowercase__       =				local_sgd_steps



      def    UpperCamelCase__	(self					:					List[str]				):

             '''simple docstring'''







             self.num_steps += 1
             if not self.enabled:
                    return

             if self.num_steps % self.local_sgd_steps == 0:
                    self._sync_and_avg_model_params()






      def    UpperCamelCase__	(self					:					Any				):

             '''simple docstring'''







             self.accelerator.wait_for_everyone()
             with self.accelerator.autocast():
                    for param in self.model.parameters():
                           lowercase__       =				self.accelerator.reduce(param.data     ,		reduction='''mean'''				)




