



'''simple docstring'''


import torch

from accelerate import Accelerator, DistributedType







class 						UpperCAmelCase						:


 '''simple docstring'''




 def __enter__( self)			->   Optional[Any]:



      """simple docstring"""







      if self.enabled:
           a_									=self.model.no_sync()
           self.model_sync_obj.__enter__()

      return self




 def __exit__( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_)			->   Optional[Any]:



      """simple docstring"""







      if self.enabled:
           # Average all models on exit
           self._sync_and_avg_model_params()
           self.model_sync_obj.__exit__(lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_)




 def __init__( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_ = True)			->   Optional[int]:



      """simple docstring"""







      if accelerator.distributed_type not in [
          DistributedType.NO,
          DistributedType.MULTI_CPU,
          DistributedType.MULTI_GPU,
      ]:
           raise NotImplementedError("LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)")
      a_									=enabled and accelerator.distributed_type != DistributedType.NO
      a_									=0
      if self.enabled:
           a_									=accelerator
           a_									=model
           a_									=local_sgd_steps




 def    lowercase_	( self)			->   Dict:



      """simple docstring"""







      self.num_steps += 1
      if not self.enabled:
           return

      if self.num_steps % self.local_sgd_steps == 0:
           self._sync_and_avg_model_params()




 def    lowercase_	( self)			->   Optional[Any]:



      """simple docstring"""







      self.accelerator.wait_for_everyone()
      with self.accelerator.autocast():
           for param in self.model.parameters():
                a_									=self.accelerator.reduce(param.data				,	reduction="mean")







'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowercase											=      {
    '''configuration_align''': [
        '''ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''AlignConfig''',
        '''AlignTextConfig''',
        '''AlignVisionConfig''',
    ],
    '''processing_align''': ['''AlignProcessor'''],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase											=      [
      '''ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''AlignModel''',
      '''AlignPreTrainedModel''',
      '''AlignTextModel''',
      '''AlignVisionModel''',
  ]

if TYPE_CHECKING:
  from .configuration_align import (
      ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP,
      AlignConfig,
      AlignTextConfig,
      AlignVisionConfig,
  )
  from .processing_align import AlignProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_align import (
        ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST,
        AlignModel,
        AlignPreTrainedModel,
        AlignTextModel,
        AlignVisionModel,
    )

else:
  import sys

  lowercase											=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


