




import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
					import torch

UpperCamelCase__  =		logging.get_logger(__name__)






def       snake_case      (							)							->							str:




	'''simple docstring'''


	return importlib.util.find_spec('deepspeed'    ) is not None


if is_accelerate_available() and is_deepspeed_available():
					from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
					# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
					# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
					from builtins import object as DeepSpeedConfig


class        __SCREAMING_SNAKE_CASE							(	lowerCamelCase						):

					def __init__(       self				, __lowercase      ):







						'''simple docstring'''




						set_hf_deepspeed_config(self      )
						dep_version_check('accelerate'      )
						dep_version_check('deepspeed'      )
						super().__init__(__lowercase      )


class        __SCREAMING_SNAKE_CASE							(	lowerCamelCase						):

					def __init__(       self				, __lowercase      ):







						'''simple docstring'''




						super().__init__(__lowercase      )
						lowerCamelCase_							:	List[str]							     =							None
						lowerCamelCase_							:	List[Any]							     =							[]

					def 			lowercase_							(       self      ):







						'''simple docstring'''




						if self._dtype is None:
							raise ValueError('trainer_config_process() wasn\'t called yet to tell dtype'      )
						return self._dtype

					def 			lowercase_							(       self				, __lowercase      ):







						'''simple docstring'''




						lowerCamelCase_							:	List[str]							     =							self.get_value(__lowercase      )
						if val is None:
							return False
						else:
							return val == "auto"

					def 			lowercase_							(       self				, __lowercase				, __lowercase				, __lowercase=None				, __lowercase=True      ):







						'''simple docstring'''




						lowerCamelCase_							:	Union[str, Any]							     =							self.find_config_node(__lowercase      )
						if config is None:
							return

						if config.get(__lowercase      ) == "auto":
							lowerCamelCase_							:	str							     =							hf_val
							return

						if not must_match:
							return

						lowerCamelCase_							:	Tuple							     =							config.get(__lowercase      )
						if ds_val is not None and ds_val != hf_val:
							self.mismatches.append(f"- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"      )

					_lowerCamelCase							=				partialmethod(lowerCamelCase     ,  must_match=lowerCamelCase						)

					def 			lowercase_							(       self				, __lowercase      ):







						'''simple docstring'''




						lowerCamelCase_							:	Union[str, Any]							     =							args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
						self.fill_match(
						    'train_micro_batch_size_per_gpu'				, args.per_device_train_batch_size				, 'per_device_train_batch_size'      )
						self.fill_match('gradient_accumulation_steps'				, args.gradient_accumulation_steps				, 'gradient_accumulation_steps'      )
						self.fill_match('train_batch_size'				, __lowercase				, 'train_batch_size (calculated)'      )
						self.fill_match('gradient_clipping'				, args.max_grad_norm				, 'max_grad_norm'      )

						self.fill_match('optimizer.params.lr'				, args.learning_rate				, 'learning_rate'      )
						self.fill_match('optimizer.params.betas'				, [args.adam_betaa, args.adam_betaa]				, 'adam_beta1+adam_beta2'      )
						self.fill_match('optimizer.params.eps'				, args.adam_epsilon				, 'adam_epsilon'      )
						self.fill_match('optimizer.params.weight_decay'				, args.weight_decay				, 'weight_decay'      )

						self.fill_only('scheduler.params.warmup_min_lr'				, 0      )  # not a trainer arg
						self.fill_match('scheduler.params.warmup_max_lr'				, args.learning_rate				, 'learning_rate'      )
						# total_num_steps - will get set in trainer_config_finalize

						# fp16
						if args.fpaa or args.fpaa_full_eval:
							lowerCamelCase_							:	List[Any]							     =							'apex' if args.fpaa_backend == 'apex' else 'amp'
						else:
							lowerCamelCase_							:	Optional[int]							     =							None

						if args.save_on_each_node:
							# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
							lowerCamelCase_							:	int							     =							self.config.get('checkpoint'				, {}      )
							lowerCamelCase_							:	List[Any]							     =							args.save_on_each_node

						# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
						# any here unless the user did the work
						self.fill_match(
						    'fp16.enabled'				, ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == 'amp')				, 'fp16|fp16_full_eval+fp16_backend(amp)'				, )

						# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
						# ZeRO features
						self.fill_match('amp.enabled'				, fpaa_backend == 'apex'				, 'fp16+fp16_backend(apex)'      )
						self.fill_match('amp.opt_level'				, args.fpaa_opt_level				, 'fp16_opt_level'      )

						self.fill_match('bf16.enabled'				, (args.bfaa or args.bfaa_full_eval)				, 'bf16|bf16_full_eval'      )

						# deepspeed's default mode is fp16 unless there is a config that says differently
						if self.is_true('bf16.enabled'      ):
							lowerCamelCase_							:	Union[str, Any]							     =							torch.bfloataa
						elif self.is_false('fp16.enabled'      ):
							lowerCamelCase_							:	List[Any]							     =							torch.floataa
						else:
							lowerCamelCase_							:	Tuple							     =							torch.floataa





					def 			lowercase_							(       self				, __lowercase				, __lowercase				, __lowercase      ):







						'''simple docstring'''




						lowerCamelCase_							:	List[Any]							     =							[
						    'zero_optimization.reduce_bucket_size',
						    'zero_optimization.stage3_prefetch_bucket_size',
						    'zero_optimization.stage3_param_persistence_threshold',
						]
						lowerCamelCase_							:	Optional[Any]							     =							[x for x in hidden_size_based_keys if self.is_auto(__lowercase      )]

						if len(__lowercase      ) > 0:
							if hasattr(model.config				, 'hidden_size'      ):
								lowerCamelCase_							:	List[Any]							     =							model.config.hidden_size
							elif hasattr(model.config				, 'hidden_sizes'      ):
								# if there are many hidden sizes pick the largest one
								lowerCamelCase_							:	str							     =							max(model.config.hidden_sizes      )
							else:
								raise ValueError(
								    'The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '
								    'therefore it\'s not possible to automatically fill out the following `auto` entries '
								    f"in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing "
								    '`auto` values for these keys with an integer value of your choice.'      )

							self.fill_only('zero_optimization.reduce_bucket_size'				, hidden_size * hidden_size      )
							if self.is_zeroa():
								# automatically assign the optimal config values based on model config
								self.fill_only('zero_optimization.stage3_prefetch_bucket_size'				, 0.9 * hidden_size * hidden_size      )
								self.fill_only('zero_optimization.stage3_param_persistence_threshold'				, 10 * hidden_size      )

        # scheduler
						self.fill_match('scheduler.params.total_num_steps'				, __lowercase				, 'num_training_steps (calculated)'      )
						self.fill_match('scheduler.params.warmup_num_steps'				, args.get_warmup_steps(__lowercase      )				, 'warmup_steps'      )

						if len(self.mismatches      ) > 0:
							lowerCamelCase_							:	Optional[int]							     =							'\n'.join(self.mismatches      )
							raise ValueError(
							    'Please correct the following DeepSpeed config values that mismatch TrainingArguments'
							    f" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'."      )
UpperCamelCase__  =		None






def       snake_case      (							A    )							->							Dict:




	'''simple docstring'''


	global _hf_deepspeed_config_weak_ref
	# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
	lowerCamelCase_							:	List[str]							     =							weakref.ref(A    )






def       snake_case      (							)							->							Tuple:




	'''simple docstring'''


	global _hf_deepspeed_config_weak_ref
	lowerCamelCase_							:	List[Any]							     =							None






def       snake_case      (							)							->							Dict:




	'''simple docstring'''


	if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
		return _hf_deepspeed_config_weak_ref().is_zeroa()
	else:
		return False






def       snake_case      (							)							->							Union[str, Any]:




	'''simple docstring'''


	if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
		return _hf_deepspeed_config_weak_ref().config
	else:
		return None






def       snake_case      (							A		,     A		,     A		,     A		,     A    )							->							int:




	'''simple docstring'''


	from accelerate.utils import DummyOptim, DummyScheduler

	lowerCamelCase_							:	Union[str, Any]							     =							hf_deepspeed_config.config

	# Optimizer + Scheduler
	# Currently supported combos:
	# 1. DS scheduler + DS optimizer: Yes
	# 2. HF scheduler + HF optimizer: Yes
	# 3. DS scheduler + HF optimizer: Yes
	# 4. HF scheduler + DS optimizer: No
	#
	# Unless Offload is enabled in which case it's:
	# 1. DS scheduler + DS optimizer: Yes
	# 2. HF scheduler + HF optimizer: Mostly*
	# 3. DS scheduler + HF optimizer: Mostly*
	# 4. HF scheduler + DS optimizer: No
	#
	# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

	lowerCamelCase_							:	int							     =							None
	if "optimizer" in config:
		if args.adafactor:
			raise ValueError(
			    '--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '
			    'Only one optimizer can be configured.'    )
		lowerCamelCase_							:	List[Any]							     =							DummyOptim(params=A    )
	else:
		if hf_deepspeed_config.is_offload():
			logger.info(
			    'Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'
			    ' custom optimizer has both CPU and GPU implementation (except LAMB)'    )

		# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
		# But trainer uses AdamW by default.
		lowerCamelCase_							:	Optional[int]							     =							trainer.create_optimizer()
		# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
		lowerCamelCase_							:	int							     =							True

	lowerCamelCase_							:	Optional[int]							     =							None
	if "scheduler" in config:
		lowerCamelCase_							:	Any							     =							DummyScheduler(A    )
	else:
		if isinstance(A		,     A    ):
			raise ValueError(
			    'Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '
			    'Please configure a scheduler in the DeepSpeed config.'    )
		lowerCamelCase_							:	str							     =							trainer.create_scheduler(num_training_steps=A		,     optimizer=A    )

	return optimizer, lr_scheduler






def       snake_case      (							A		,     A		,     A=False    )							->							Union[str, Any]:




	'''simple docstring'''


	from deepspeed.utils import logger as ds_logger

	lowerCamelCase_							:	Optional[int]							     =							trainer.model
	lowerCamelCase_							:	Any							     =							trainer.args

	lowerCamelCase_							:	List[str]							     =							trainer.accelerator.state.deepspeed_plugin.hf_ds_config

	# resume config update - some bits like `model` and `num_training_steps` only become available during train
	hf_deepspeed_config.trainer_config_finalize(A		,     A		,     A    )

	# set the Deepspeed log level consistent with the Trainer
	ds_logger.setLevel(args.get_process_log_level()    )

	if inference:
		# only Z3 makes sense for the inference
		if not hf_deepspeed_config.is_zeroa():
			raise ValueError('ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'    )

		# in case the training config is re-used for inference
		hf_deepspeed_config.del_config_sub_tree('optimizer'    )
		hf_deepspeed_config.del_config_sub_tree('lr_scheduler'    )
		lowerCamelCase_							:	Optional[Any]							     =							None, None
		lowerCamelCase_							:	List[Any]							     =							None
	else:
		lowerCamelCase_							:	Union[str, Any]							     =							None  # important for when deepspeed_init is used as re-init
		lowerCamelCase_							:	str							     =							list(filter(lambda A    : p.requires_grad		,     model.parameters()    )    )
		lowerCamelCase_							:	Union[str, Any]							     =							deepspeed_optim_sched(
		    A		,     A		,     A		,     A		,     A    )

	# keep for quick debug:
	# from pprint import pprint; pprint(config)

	return optimizer, lr_scheduler






def       snake_case      (							A		,     A    )							->							Tuple:




	'''simple docstring'''


	import glob

	lowerCamelCase_							:	Optional[Any]							     =							sorted(glob.glob(f"{checkpoint_path}/global_step*"    )    )

	if len(A    ) > 0:
		logger.info(f"Attempting to resume from {checkpoint_path}"    )
		# this magically updates self.optimizer and self.lr_scheduler
		lowerCamelCase_							:	Union[str, Any]							     =							deepspeed_engine.load_checkpoint(
		    A		,     load_optimizer_states=A		,     load_lr_scheduler_states=A    )
		if load_path is None:
			raise ValueError(f"[deepspeed] failed to resume from checkpoint {checkpoint_path}"    )
	else:
		raise ValueError(f"Can't find a valid checkpoint at {checkpoint_path}"    )






from dataclasses import dataclass
from typing import List, Optional, Union

import numpy as np
import PIL
from PIL import Image

from ...utils import is_torch_available, is_transformers_available


if is_transformers_available() and is_torch_available():
							from .image_encoder import PaintByExampleImageEncoder
							from .pipeline_paint_by_example import PaintByExamplePipeline
