


import torch

from accelerate import Accelerator, DistributedType






class        UpperCAmelCase__			:



    """simple docstring"""






    def __enter__(     self       )    ->			Optional[Any]:







         if self.enabled:
              __UpperCamelCase       				=self.model.no_sync()
              self.model_sync_obj.__enter__()

         return self






    def __exit__(     self   ,  A_   ,  A_   ,  A_       )    ->			Optional[Any]:







         if self.enabled:
              # Average all models on exit
              self._sync_and_avg_model_params()
              self.model_sync_obj.__exit__(lowerCAmelCase_   ,  lowerCAmelCase_   ,  lowerCAmelCase_       )






    def __init__(     self   ,  A_   ,  A_   ,  A_   ,  A_ = True       )    ->			Optional[int]:







         if accelerator.distributed_type not in [
             DistributedType.NO,
             DistributedType.MULTI_CPU,
             DistributedType.MULTI_GPU,
         ]:
              raise NotImplementedError('LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'       )
         __UpperCamelCase       				=enabled and accelerator.distributed_type != DistributedType.NO
         __UpperCamelCase       				=0
         if self.enabled:
              __UpperCamelCase       				=accelerator
              __UpperCamelCase       				=model
              __UpperCamelCase       				=local_sgd_steps






    def _a   (     self       )    ->			Dict:
         self.num_steps += 1
         if not self.enabled:
              return

         if self.num_steps % self.local_sgd_steps == 0:
              self._sync_and_avg_model_params()






    def _a   (     self       )    ->			Optional[Any]:
         self.accelerator.wait_for_everyone()
         with self.accelerator.autocast():
              for param in self.model.parameters():
                   __UpperCamelCase       				=self.accelerator.reduce(param.data   ,  reduction='mean'       )










'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowercase											=      {
    '''configuration_align''': [
        '''ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''AlignConfig''',
        '''AlignTextConfig''',
        '''AlignVisionConfig''',
    ],
    '''processing_align''': ['''AlignProcessor'''],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase											=      [
      '''ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''AlignModel''',
      '''AlignPreTrainedModel''',
      '''AlignTextModel''',
      '''AlignVisionModel''',
  ]

if TYPE_CHECKING:
  from .configuration_align import (
      ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP,
      AlignConfig,
      AlignTextConfig,
      AlignVisionConfig,
  )
  from .processing_align import AlignProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_align import (
        ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST,
        AlignModel,
        AlignPreTrainedModel,
        AlignTextModel,
        AlignVisionModel,
    )

else:
  import sys

  lowercase											=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


