from __future__ import annotations

import inspect
import unittest

from transformers import MobileViTConfig
from transformers.file_utils import is_tf_available, is_vision_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import numpy as np
	import tensorflow as tf

	from transformers import TFMobileViTForImageClassification, TFMobileViTForSemanticSegmentation, TFMobileViTModel
	from transformers.models.mobilevit.modeling_tf_mobilevit import TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
	from PIL import Image

	from transformers import MobileViTImageProcessor




class      lowercase_						(						UpperCamelCase_		):



	"""simple docstring"""





	def     SCREAMING_SNAKE_CASE_       (							self       )		->List[Any]:
								lowerCAmelCase				=	self.config_class(**self.inputs_dict       )
								self.parent.assertTrue(hasattr(__SCREAMING_SNAKE_CASE	,    '''hidden_sizes'''       )       )
								self.parent.assertTrue(hasattr(__SCREAMING_SNAKE_CASE	,    '''neck_hidden_sizes'''       )       )
								self.parent.assertTrue(hasattr(__SCREAMING_SNAKE_CASE	,    '''num_attention_heads'''       )       )




class      lowercase_						:



	"""simple docstring"""





	def __init__(							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE=13	,    __SCREAMING_SNAKE_CASE=32	,    __SCREAMING_SNAKE_CASE=2	,    __SCREAMING_SNAKE_CASE=3	,    __SCREAMING_SNAKE_CASE=640	,    __SCREAMING_SNAKE_CASE=4	,    __SCREAMING_SNAKE_CASE="silu"	,    __SCREAMING_SNAKE_CASE=3	,    __SCREAMING_SNAKE_CASE=32	,    __SCREAMING_SNAKE_CASE=0.1	,    __SCREAMING_SNAKE_CASE=0.1	,    __SCREAMING_SNAKE_CASE=0.1	,    __SCREAMING_SNAKE_CASE=0.0_2	,    __SCREAMING_SNAKE_CASE=True	,    __SCREAMING_SNAKE_CASE=True	,    __SCREAMING_SNAKE_CASE=10	,    __SCREAMING_SNAKE_CASE=None	,    )		->Optional[int]:
								lowerCAmelCase				=	parent
								lowerCAmelCase				=	batch_size
								lowerCAmelCase				=	image_size
								lowerCAmelCase				=	patch_size
								lowerCAmelCase				=	num_channels
								lowerCAmelCase				=	last_hidden_size
								lowerCAmelCase				=	num_attention_heads
								lowerCAmelCase				=	hidden_act
								lowerCAmelCase				=	conv_kernel_size
								lowerCAmelCase				=	output_stride
								lowerCAmelCase				=	hidden_dropout_prob
								lowerCAmelCase				=	attention_probs_dropout_prob
								lowerCAmelCase				=	classifier_dropout_prob
								lowerCAmelCase				=	use_labels
								lowerCAmelCase				=	is_training
								lowerCAmelCase				=	num_labels
								lowerCAmelCase				=	initializer_range
								lowerCAmelCase				=	scope


	def     SCREAMING_SNAKE_CASE_       (							self       )		->List[Any]:
								lowerCAmelCase				=	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]       )

								lowerCAmelCase				=	None
								lowerCAmelCase				=	None
								if self.use_labels:
															lowerCAmelCase				=	ids_tensor([self.batch_size]	,    self.num_labels       )
															lowerCAmelCase				=	ids_tensor([self.batch_size, self.image_size, self.image_size]	,    self.num_labels       )

								lowerCAmelCase				=	self.get_config()

								return config, pixel_values, labels, pixel_labels


	def     SCREAMING_SNAKE_CASE_       (							self       )		->Dict:
								return MobileViTConfig(
								    image_size=self.image_size	,    patch_size=self.patch_size	,    num_channels=self.num_channels	,    num_attention_heads=self.num_attention_heads	,    hidden_act=self.hidden_act	,    conv_kernel_size=self.conv_kernel_size	,    output_stride=self.output_stride	,    hidden_dropout_prob=self.hidden_dropout_prob	,    attention_probs_dropout_prob=self.attention_probs_dropout_prob	,    classifier_dropout_prob=self.classifier_dropout_prob	,    initializer_range=self.initializer_range	,    )


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->Optional[Any]:
								lowerCAmelCase				=	TFMobileViTModel(config=__SCREAMING_SNAKE_CASE       )
								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    training=__SCREAMING_SNAKE_CASE       )
								lowerCAmelCase				=	lowerCAmelCase				=	self.image_size // self.output_stride
								self.parent.assertEqual(
								    result.last_hidden_state.shape	,    (self.batch_size, self.last_hidden_size, expected_height, expected_width)       )


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->Optional[int]:
								lowerCAmelCase				=	self.num_labels
								lowerCAmelCase				=	TFMobileViTForImageClassification(__SCREAMING_SNAKE_CASE       )
								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE	,    training=__SCREAMING_SNAKE_CASE       )
								self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_labels)       )


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->Union[str, Any]:
								lowerCAmelCase				=	self.num_labels
								lowerCAmelCase				=	TFMobileViTForSemanticSegmentation(__SCREAMING_SNAKE_CASE       )
								lowerCAmelCase				=	lowerCAmelCase				=	self.image_size // self.output_stride

								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    training=__SCREAMING_SNAKE_CASE       )
								self.parent.assertEqual(
								    result.logits.shape	,    (self.batch_size, self.num_labels, expected_height, expected_width)       )

								lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    labels=__SCREAMING_SNAKE_CASE	,    training=__SCREAMING_SNAKE_CASE       )
								self.parent.assertEqual(
								    result.logits.shape	,    (self.batch_size, self.num_labels, expected_height, expected_width)       )




	def     SCREAMING_SNAKE_CASE_       (							self       )		->List[str]:
								lowerCAmelCase				=	self.prepare_config_and_inputs()
								lowerCAmelCase	,   lowerCAmelCase	,   lowerCAmelCase	,   lowerCAmelCase				=	config_and_inputs
								lowerCAmelCase				=	{'''pixel_values''': pixel_values}
								return config, inputs_dict




@require_tf
class      lowercase_						(						UpperCamelCase_   ,  UpperCamelCase_   ,  unittest.TestCase		):



	"""simple docstring"""





	UpperCAmelCase_						:      str     =       (
	    (TFMobileViTModel, TFMobileViTForImageClassification, TFMobileViTForSemanticSegmentation)
	    if is_tf_available()
	    else ()
	)
	UpperCAmelCase_						:      Dict     =       (
	    {"""feature-extraction""": TFMobileViTModel, """image-classification""": TFMobileViTForImageClassification}
	    if is_tf_available()
	    else {}
	)

	UpperCAmelCase_						:      Optional[int]     =       False
	UpperCAmelCase_						:      Optional[Any]     =       False
	UpperCAmelCase_						:      str     =       False
	UpperCAmelCase_						:      Optional[int]     =       False
	UpperCAmelCase_						:      Union[str, Any]     =       False


	def     SCREAMING_SNAKE_CASE_       (							self       )		->Optional[int]:
								lowerCAmelCase				=	TFMobileViTModelTester(self       )
								lowerCAmelCase				=	TFMobileViTConfigTester(self	,    config_class=__SCREAMING_SNAKE_CASE	,    has_text_modality=__SCREAMING_SNAKE_CASE       )


	def     SCREAMING_SNAKE_CASE_       (							self       )		->List[str]:
								self.config_tester.run_common_tests()


	@unittest.skip(reason='''MobileViT does not use inputs_embeds'''       )
	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
								pass


	@unittest.skip(reason='''MobileViT does not support input and output embeddings'''       )
	def     SCREAMING_SNAKE_CASE_       (							self       )		->Any:
								pass


	@unittest.skip(reason='''MobileViT does not output attentions'''       )
	def     SCREAMING_SNAKE_CASE_       (							self       )		->Optional[int]:
								pass


	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
								lowerCAmelCase	,   lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
															lowerCAmelCase				=	model_class(__SCREAMING_SNAKE_CASE       )
															lowerCAmelCase				=	inspect.signature(model.call       )
															# signature.parameters is an OrderedDict => so arg_names order is deterministic
															lowerCAmelCase				=	[*signature.parameters.keys()]

															lowerCAmelCase				=	['''pixel_values''']
															self.assertListEqual(arg_names[:1]	,    __SCREAMING_SNAKE_CASE       )


	def     SCREAMING_SNAKE_CASE_       (							self       )		->Optional[int]:
								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE       )


	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
								def check_hidden_states_output(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       ):
															lowerCAmelCase				=	model_class(__SCREAMING_SNAKE_CASE       )

															lowerCAmelCase				=	model(**self._prepare_for_class(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )       )

															lowerCAmelCase				=	outputs.hidden_states

															lowerCAmelCase				=	5
															self.assertEqual(len(__SCREAMING_SNAKE_CASE       )	,    __SCREAMING_SNAKE_CASE       )

															# MobileViT's feature maps are of shape (batch_size, num_channels, height, width)
															# with the width and height being successively divided by 2.
															lowerCAmelCase				=	2
															for i in range(len(__SCREAMING_SNAKE_CASE       )       ):
																						self.assertListEqual(
																						    list(hidden_states[i].shape[-2:]       )	,    [self.model_tester.image_size // divisor, self.model_tester.image_size // divisor]	,    )
																						divisor *= 2

															self.assertEqual(self.model_tester.output_stride	,    divisor // 2       )

								lowerCAmelCase	,   lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
															lowerCAmelCase				=	True
															check_hidden_states_output(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )

															# check that output_hidden_states also work using config
															del inputs_dict["output_hidden_states"]
															lowerCAmelCase				=	True

															check_hidden_states_output(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )


	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_image_classification(*__SCREAMING_SNAKE_CASE       )


	def     SCREAMING_SNAKE_CASE_       (							self       )		->Dict:
								lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_semantic_segmentation(*__SCREAMING_SNAKE_CASE       )


	@unittest.skipIf(
	    not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''       )       ) == 0	,    reason='''TF does not support backprop for grouped convolutions on CPU.'''	,    )
	def     SCREAMING_SNAKE_CASE_       (							self       )		->Dict:
								super().test_dataset_conversion()


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE=2e-1	,    __SCREAMING_SNAKE_CASE=2e-1       )		->List[str]:
								self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    atol=__SCREAMING_SNAKE_CASE	,    rtol=__SCREAMING_SNAKE_CASE       )       )


	@unittest.skipIf(
	    not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''       )       ) == 0	,    reason='''TF does not support backprop for grouped convolutions on CPU.'''	,    )
	@slow
	def     SCREAMING_SNAKE_CASE_       (							self       )		->Union[str, Any]:
								lowerCAmelCase	,   lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
															# Since `TFMobileViTModel` cannot operate with the default `fit()` method.
															if model_class.__name__ != "TFMobileViTModel":
																						lowerCAmelCase				=	model_class(__SCREAMING_SNAKE_CASE       )
																						if getattr(__SCREAMING_SNAKE_CASE	,    '''hf_compute_loss'''	,    __SCREAMING_SNAKE_CASE       ):
																													super().test_keras_fit()


	def     SCREAMING_SNAKE_CASE_       (							self       )		->Dict:
								lowerCAmelCase	,   lowerCAmelCase				=	self.model_tester.prepare_config_and_inputs_for_common()
								for model_class in self.all_model_classes:
															# set an ignore index to correctly test the masked loss used in
															# `TFMobileViTForSemanticSegmentation`.
															if model_class.__name__ != "TFMobileViTForSemanticSegmentation":
																						lowerCAmelCase				=	5

															lowerCAmelCase				=	model_class(__SCREAMING_SNAKE_CASE       )
															if getattr(__SCREAMING_SNAKE_CASE	,    '''hf_compute_loss'''	,    __SCREAMING_SNAKE_CASE       ):
																						# The number of elements in the loss should be the same as the number of elements in the label
																						lowerCAmelCase				=	self._prepare_for_class(inputs_dict.copy()	,    __SCREAMING_SNAKE_CASE	,    return_labels=__SCREAMING_SNAKE_CASE       )
																						lowerCAmelCase				=	prepared_for_class[
																						    sorted(prepared_for_class.keys() - inputs_dict.keys()	,    reverse=__SCREAMING_SNAKE_CASE       )[0]
																						]
																						lowerCAmelCase				=	added_label.shape.as_list()[:1]

																						# Test that model correctly compute the loss with kwargs
																						lowerCAmelCase				=	self._prepare_for_class(inputs_dict.copy()	,    __SCREAMING_SNAKE_CASE	,    return_labels=__SCREAMING_SNAKE_CASE       )
																						lowerCAmelCase				=	{'''input_ids''', '''pixel_values''', '''input_features'''}
																						lowerCAmelCase				=	possible_input_names.intersection(set(__SCREAMING_SNAKE_CASE       )       ).pop()
																						lowerCAmelCase				=	prepared_for_class.pop(__SCREAMING_SNAKE_CASE       )

																						lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    **__SCREAMING_SNAKE_CASE       )[0]
																						self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]       )

																						# Test that model correctly compute the loss when we mask some positions
																						lowerCAmelCase				=	self._prepare_for_class(inputs_dict.copy()	,    __SCREAMING_SNAKE_CASE	,    return_labels=__SCREAMING_SNAKE_CASE       )
																						lowerCAmelCase				=	{'''input_ids''', '''pixel_values''', '''input_features'''}
																						lowerCAmelCase				=	possible_input_names.intersection(set(__SCREAMING_SNAKE_CASE       )       ).pop()
																						lowerCAmelCase				=	prepared_for_class.pop(__SCREAMING_SNAKE_CASE       )
																						if "labels" in prepared_for_class:
																													lowerCAmelCase				=	prepared_for_class['''labels'''].numpy()
																													if len(labels.shape       ) > 1 and labels.shape[1] != 1:
																																				# labels[0] = -100
																																				lowerCAmelCase				=	tf.convert_to_tensor(__SCREAMING_SNAKE_CASE       )
																																				lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE	,    **__SCREAMING_SNAKE_CASE       )[0]
																																				self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]       )
																																				self.assertTrue(not np.any(np.isnan(loss.numpy()       )       )       )

                # Test that model correctly compute the loss with a dict
																						lowerCAmelCase				=	self._prepare_for_class(inputs_dict.copy()	,    __SCREAMING_SNAKE_CASE	,    return_labels=__SCREAMING_SNAKE_CASE       )
																						lowerCAmelCase				=	model(__SCREAMING_SNAKE_CASE       )[0]
																						self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]       )

																						# Test that model correctly compute the loss with a tuple
																						lowerCAmelCase				=	self._prepare_for_class(inputs_dict.copy()	,    __SCREAMING_SNAKE_CASE	,    return_labels=__SCREAMING_SNAKE_CASE       )

																						# Get keys that were added with the _prepare_for_class function
																						lowerCAmelCase				=	prepared_for_class.keys() - inputs_dict.keys()
																						lowerCAmelCase				=	inspect.signature(model.call       ).parameters
																						lowerCAmelCase				=	list(signature.keys()       )

																						# Create a dictionary holding the location of the tensors in the tuple
																						lowerCAmelCase				=	{0: input_name}
																						for label_key in label_keys:
																													lowerCAmelCase				=	signature_names.index(__SCREAMING_SNAKE_CASE       )
																													lowerCAmelCase				=	label_key
																						lowerCAmelCase				=	sorted(tuple_index_mapping.items()       )
																						# Initialize a list with their default values, update the values and convert to a tuple
																						lowerCAmelCase				=	[]

																						for name in signature_names:
																													if name != "kwargs":
																																				list_input.append(signature[name].default       )

																						for index, value in sorted_tuple_index_mapping:
																													lowerCAmelCase				=	prepared_for_class[value]

																						lowerCAmelCase				=	tuple(__SCREAMING_SNAKE_CASE       )

																						# Send to model
																						lowerCAmelCase				=	model(tuple_input[:-1]       )[0]

																						self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]       )




	@slow
	def     SCREAMING_SNAKE_CASE_       (							self       )		->List[Any]:
								for model_name in TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															lowerCAmelCase				=	TFMobileViTModel.from_pretrained(__SCREAMING_SNAKE_CASE       )
															self.assertIsNotNone(__SCREAMING_SNAKE_CASE       )

def      SCREAMING_SNAKE_CASE_  (		)  ->			List[str]:
							lowerCAmelCase				=	Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''							)
							return image






@require_tf
class      lowercase_						(						unittest.TestCase		):



	"""simple docstring"""





	@slow
	def     SCREAMING_SNAKE_CASE_       (							self       )		->Dict:
								lowerCAmelCase				=	TFMobileViTForImageClassification.from_pretrained('''apple/mobilevit-xx-small'''       )

								lowerCAmelCase				=	MobileViTImageProcessor.from_pretrained('''apple/mobilevit-xx-small'''       )
								lowerCAmelCase				=	prepare_img()
								lowerCAmelCase				=	image_processor(images=__SCREAMING_SNAKE_CASE	,    return_tensors='''tf'''       )

								# forward pass
								lowerCAmelCase				=	model(**__SCREAMING_SNAKE_CASE	,    training=__SCREAMING_SNAKE_CASE       )

								# verify the logits
								lowerCAmelCase				=	tf.TensorShape((1, 1000)       )
								self.assertEqual(outputs.logits.shape	,    __SCREAMING_SNAKE_CASE       )

								lowerCAmelCase				=	tf.constant([-1.9_3_6_4, -1.2_3_2_7, -0.4_6_5_3]       )

								tf.debugging.assert_near(outputs.logits[0, :3]	,    __SCREAMING_SNAKE_CASE	,    atol=1e-4	,    rtol=1e-04       )




	@slow
	def     SCREAMING_SNAKE_CASE_       (							self       )		->str:
								# `from_pt` will be removed
								lowerCAmelCase				=	TFMobileViTForSemanticSegmentation.from_pretrained('''apple/deeplabv3-mobilevit-xx-small'''       )

								lowerCAmelCase				=	MobileViTImageProcessor.from_pretrained('''apple/deeplabv3-mobilevit-xx-small'''       )

								lowerCAmelCase				=	prepare_img()
								lowerCAmelCase				=	image_processor(images=__SCREAMING_SNAKE_CASE	,    return_tensors='''tf'''       )

								# forward pass
								lowerCAmelCase				=	model(inputs.pixel_values	,    training=__SCREAMING_SNAKE_CASE       )
								lowerCAmelCase				=	outputs.logits

								# verify the logits
								lowerCAmelCase				=	tf.TensorShape((1, 21, 32, 32)       )
								self.assertEqual(logits.shape	,    __SCREAMING_SNAKE_CASE       )

								lowerCAmelCase				=	tf.constant(
								    [
								        [[6.9_7_1_3, 6.9_7_8_6, 7.2_4_2_2], [7.2_8_9_3, 7.2_8_2_5, 7.4_4_4_6], [7.6_5_8_0, 7.8_7_9_7, 7.9_4_2_0]],
								        [[-1_0.6_8_6_9, -1_0.3_2_5_0, -1_0.3_4_7_1], [-1_0.4_2_2_8, -9.9_8_6_8, -9.7_1_3_2], [-1_1.0_4_0_5, -1_1.0_2_2_1, -1_0.7_3_1_8]],
								        [[-3.3_0_8_9, -2.8_5_3_9, -2.6_7_4_0], [-3.2_7_0_6, -2.5_6_2_1, -2.5_1_0_8], [-3.2_5_3_4, -2.6_6_1_5, -2.6_6_5_1]],
								    ]       )

								tf.debugging.assert_near(logits[0, :3, :3, :3]	,    __SCREAMING_SNAKE_CASE	,    rtol=1e-4	,    atol=1e-4       )




import torch

from accelerate import Accelerator, DistributedType




class      lowercase_						:



	"""simple docstring"""





	def __enter__(							self       )		->Dict:
								if self.enabled:
															lowerCAmelCase				=	self.model.no_sync()
															self.model_sync_obj.__enter__()

								return self


	def __exit__(							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->Any:
								if self.enabled:
															# Average all models on exit
															self._sync_and_avg_model_params()
															self.model_sync_obj.__exit__(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )


	def __init__(							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE = True       )		->Tuple:
								if accelerator.distributed_type not in [
								    DistributedType.NO,
								    DistributedType.MULTI_CPU,
								    DistributedType.MULTI_GPU,
								]:
															raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''       )
								lowerCAmelCase				=	enabled and accelerator.distributed_type != DistributedType.NO
								lowerCAmelCase				=	0
								if self.enabled:
															lowerCAmelCase				=	accelerator
															lowerCAmelCase				=	model
															lowerCAmelCase				=	local_sgd_steps


	def     SCREAMING_SNAKE_CASE_       (							self       )		->List[Any]:
								self.num_steps += 1
								if not self.enabled:
															return

								if self.num_steps % self.local_sgd_steps == 0:
															self._sync_and_avg_model_params()




	def     SCREAMING_SNAKE_CASE_       (							self       )		->Optional[int]:
								self.accelerator.wait_for_everyone()
								with self.accelerator.autocast():
															for param in self.model.parameters():
																						lowerCAmelCase				=	self.accelerator.reduce(param.data	,    reduction='''mean'''       )



