'''simple docstring'''
import torch

from accelerate import Accelerator, DistributedType




class _lowerCAmelCase			:







      def __enter__(  self     ):
        if self.enabled:
          lowercase				:							str									=   self.model.no_sync()
          self.model_sync_obj.__enter__()

        return self







      def __exit__(  self  ,    lowercase  ,    lowercase  ,    lowercase     ):
        if self.enabled:
          # Average all models on exit
          self._sync_and_avg_model_params()
          self.model_sync_obj.__exit__(_lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase     )







      def __init__(  self  ,    lowercase  ,    lowercase  ,    lowercase  ,    lowercase = True     ):
        if accelerator.distributed_type not in [
            DistributedType.NO,
            DistributedType.MULTI_CPU,
            DistributedType.MULTI_GPU,
        ]:
          raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''     )
        lowercase				:							Dict									=   enabled and accelerator.distributed_type != DistributedType.NO
        lowercase				:							Optional[Any]									=   0
        if self.enabled:
          lowercase				:							int									=   accelerator
          lowercase				:							int									=   model
          lowercase				:							List[str]									=   local_sgd_steps







      def      lowerCamelCase__      (  self     ):
        self.num_steps += 1
        if not self.enabled:
          return

        if self.num_steps % self.local_sgd_steps == 0:
          self._sync_and_avg_model_params()







      def      lowerCamelCase__      (  self     ):
        self.accelerator.wait_for_everyone()
        with self.accelerator.autocast():
          for param in self.model.parameters():
            lowercase				:							Any									=   self.accelerator.reduce(param.data  ,    reduction='''mean'''     )
'''simple docstring'''
from .prime_check import Test

Test()



