




"""simple docstring"""





from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import Dataset

from .deepspeed import is_deepspeed_zeroa_enabled
from .generation.configuration_utils import GenerationConfig
from .trainer import Trainer
from .utils import logging


if TYPE_CHECKING:
							from .data.data_collator import DataCollator
							from .modeling_utils import PreTrainedModel
							from .tokenization_utils_base import PreTrainedTokenizerBase
							from .trainer_callback import TrainerCallback
							from .trainer_utils import EvalPrediction, PredictionOutput
							from .training_args import TrainingArguments


_lowercase	      =    logging.get_logger(__name__)

class       __lowercase       (			snake_case_	):


				'''simple docstring'''


				def __init__(   self   :     Union[str, Any]      ,SCREAMING_SNAKE_CASE_   :     Union["PreTrainedModel", nn.Module] = None      ,SCREAMING_SNAKE_CASE_   :     "TrainingArguments" = None      ,SCREAMING_SNAKE_CASE_   :     Optional["DataCollator"] = None      ,SCREAMING_SNAKE_CASE_   :     Optional[Dataset] = None      ,SCREAMING_SNAKE_CASE_   :     Optional[Union[Dataset, Dict[str, Dataset]]] = None      ,SCREAMING_SNAKE_CASE_   :     Optional["PreTrainedTokenizerBase"] = None      ,SCREAMING_SNAKE_CASE_   :     Optional[Callable[[], "PreTrainedModel"]] = None      ,SCREAMING_SNAKE_CASE_   :     Optional[Callable[["EvalPrediction"], Dict]] = None      ,SCREAMING_SNAKE_CASE_   :     Optional[List["TrainerCallback"]] = None      ,SCREAMING_SNAKE_CASE_   :     Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None)      ,SCREAMING_SNAKE_CASE_   :     Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None      ,):
										super().__init__(
										    model=SCREAMING_SNAKE_CASE_      ,args=SCREAMING_SNAKE_CASE_      ,data_collator=SCREAMING_SNAKE_CASE_      ,train_dataset=SCREAMING_SNAKE_CASE_      ,eval_dataset=SCREAMING_SNAKE_CASE_      ,tokenizer=SCREAMING_SNAKE_CASE_      ,model_init=SCREAMING_SNAKE_CASE_      ,compute_metrics=SCREAMING_SNAKE_CASE_      ,callbacks=SCREAMING_SNAKE_CASE_      ,optimizers=SCREAMING_SNAKE_CASE_      ,preprocess_logits_for_metrics=SCREAMING_SNAKE_CASE_      ,)

										# Override self.model.generation_config if a GenerationConfig is specified in args.
										# Priority: args.generation_config > model.generation_config > default GenerationConfig.
										if self.args.generation_config is not None:
																snake_case__       :					List[Any]										=					self.load_generation_config(self.args.generation_config      )
																snake_case__       :					Optional[int]										=					gen_config

				@staticmethod
				def       _lowerCamelCase					(   SCREAMING_SNAKE_CASE_   :     Union[str, GenerationConfig]      ):

										# GenerationConfig provided, nothing to do
										if isinstance(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ):
																return deepcopy(SCREAMING_SNAKE_CASE_      )

										# str or Path
										snake_case__       :					Optional[int]										=					Path(SCREAMING_SNAKE_CASE_      ) if isinstance(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ) else gen_config_arg
										snake_case__       :					Optional[Any]										=					None

										# Figuring if it is path pointing to a file, pointing to a directory or else a model id or URL
										# This step is required in order to determine config_file_name
										if pretrained_model_name.is_file():
																snake_case__       :					List[Any]										=					pretrained_model_name.name
																snake_case__       :					Any										=					pretrained_model_name.parent
										# dir path
										elif pretrained_model_name.is_dir():
																pass
										# model id or URL
										else:
																snake_case__       :					Optional[Any]										=					gen_config_arg

										snake_case__       :					Optional[Any]										=					GenerationConfig.from_pretrained(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      )
										return gen_config

				def       _lowerCamelCase					(   self   :     List[str]      ,SCREAMING_SNAKE_CASE_   :     Optional[Dataset] = None      ,SCREAMING_SNAKE_CASE_   :     Optional[List[str]] = None      ,SCREAMING_SNAKE_CASE_   :     str = "eval"      ,**SCREAMING_SNAKE_CASE_   :     int      ,):
										snake_case__       :					List[str]										=					gen_kwargs.copy()
										if gen_kwargs.get('''max_length'''      ) is None and gen_kwargs.get('''max_new_tokens'''      ) is None:
																snake_case__       :					Optional[int]										=					self.args.generation_max_length
										snake_case__       :					Optional[Any]										=					(
										    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''      ) is not None else self.args.generation_num_beams
										)
										snake_case__       :					Optional[Any]										=					gen_kwargs

										return super().evaluate(SCREAMING_SNAKE_CASE_      ,ignore_keys=SCREAMING_SNAKE_CASE_      ,metric_key_prefix=SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     List[Any]      ,SCREAMING_SNAKE_CASE_   :     Dataset      ,SCREAMING_SNAKE_CASE_   :     Optional[List[str]] = None      ,SCREAMING_SNAKE_CASE_   :     str = "test"      ,**SCREAMING_SNAKE_CASE_   :     int      ,):
										snake_case__       :					Union[str, Any]										=					gen_kwargs.copy()
										if gen_kwargs.get('''max_length'''      ) is None and gen_kwargs.get('''max_new_tokens'''      ) is None:
																snake_case__       :					Any										=					self.args.generation_max_length
										snake_case__       :					str										=					(
										    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''      ) is not None else self.args.generation_num_beams
										)
										snake_case__       :					List[str]										=					gen_kwargs

										return super().predict(SCREAMING_SNAKE_CASE_      ,ignore_keys=SCREAMING_SNAKE_CASE_      ,metric_key_prefix=SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     nn.Module      ,SCREAMING_SNAKE_CASE_   :     Dict[str, Union[torch.Tensor, Any]]      ,SCREAMING_SNAKE_CASE_   :     bool      ,SCREAMING_SNAKE_CASE_   :     Optional[List[str]] = None      ,**SCREAMING_SNAKE_CASE_   :     List[Any]      ,):

										if not self.args.predict_with_generate or prediction_loss_only:
																return super().prediction_step(
																    SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,prediction_loss_only=SCREAMING_SNAKE_CASE_      ,ignore_keys=SCREAMING_SNAKE_CASE_      )

										snake_case__       :					List[Any]										=					'''labels''' in inputs
										snake_case__       :					Tuple										=					self._prepare_inputs(SCREAMING_SNAKE_CASE_      )

										# XXX: adapt synced_gpus for fairscale as well
										# Priority (handled in generate):
										# gen_kwargs > model.generation_config > default GenerationConfig()

										if len(SCREAMING_SNAKE_CASE_      ) == 0 and hasattr(self      ,'''_gen_kwargs'''      ):
																snake_case__       :					Any										=					self._gen_kwargs.copy()

										if gen_kwargs.get('''max_length'''      ) is None and gen_kwargs.get('''max_new_tokens'''      ) is None:
																snake_case__       :					Any										=					self.model.config.max_length
										snake_case__       :					int										=					(
										    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''      ) is not None else self.model.config.num_beams
										)
										snake_case__       :					Optional[int]										=					True if is_deepspeed_zeroa_enabled() else False
										snake_case__       :					Any										=					(
										    gen_kwargs['''synced_gpus'''] if gen_kwargs.get('''synced_gpus'''      ) is not None else default_synced_gpus
										)

										# If the `decoder_input_ids` was created from `labels`, evict the former, so that the model can freely generate
										# (otherwise, it would continue generating from the padded `decoder_input_ids`)
										if (
										    "labels" in inputs
										    and "decoder_input_ids" in inputs
										    and inputs["labels"].shape == inputs["decoder_input_ids"].shape
										):
																snake_case__       :					List[str]										=					{k: v for k, v in inputs.items() if k != '''decoder_input_ids'''}
										snake_case__       :					List[str]										=					self.model.generate(**SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_      )

										# Temporary hack to ensure the generation config is not initialized for each iteration of the evaluation loop
										# TODO: remove this hack when the legacy code that initializes generation_config from a model config is
										# removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183
										if self.model.generation_config._from_model_config:
																snake_case__       :					int										=					False

										# Retrieves GenerationConfig from model.generation_config
										snake_case__       :					Dict										=					self.model.generation_config
										# in case the batch is shorter than max length, the output should be padded
										if generated_tokens.shape[-1] < gen_config.max_length:
																snake_case__       :					Tuple										=					self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE_      ,gen_config.max_length      )
										elif gen_config.max_new_tokens is not None and generated_tokens.shape[-1] < gen_config.max_new_tokens + 1:
																snake_case__       :					Any										=					self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE_      ,gen_config.max_new_tokens + 1      )

										with torch.no_grad():
																if has_labels:
																						with self.compute_loss_context_manager():
																												snake_case__       :					Union[str, Any]										=					model(**SCREAMING_SNAKE_CASE_      )
																						if self.label_smoother is not None:
																												snake_case__       :					str										=					self.label_smoother(SCREAMING_SNAKE_CASE_      ,inputs['''labels''']      ).mean().detach()
																						else:
																												snake_case__       :					Optional[Any]										=					(outputs['''loss'''] if isinstance(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ) else outputs[0]).mean().detach()
																else:
																						snake_case__       :					Optional[Any]										=					None

										if self.args.prediction_loss_only:
																return loss, None, None

										if has_labels:
																snake_case__       :					Tuple										=					inputs['''labels''']
																if labels.shape[-1] < gen_config.max_length:
																						snake_case__       :					List[str]										=					self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE_      ,gen_config.max_length      )
																elif gen_config.max_new_tokens is not None and labels.shape[-1] < gen_config.max_new_tokens + 1:
																						snake_case__       :					Optional[int]										=					self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE_      ,gen_config.max_new_tokens + 1      )
										else:
																snake_case__       :					List[str]										=					None

										return loss, generated_tokens, labels





				def       _lowerCamelCase					(   self   :     Dict      ,SCREAMING_SNAKE_CASE_   :     Dict      ,SCREAMING_SNAKE_CASE_   :     Dict      ):
										if self.tokenizer is not None and hasattr(self.tokenizer      ,'''pad_token_id'''      ):
																# If PAD token is not defined at least EOS token has to be defined
																snake_case__       :					List[str]										=					(
																    self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
																)
										else:
																if self.model.config.pad_token_id is not None:
																						snake_case__       :					Any										=					self.model.config.pad_token_id
																else:
																						raise ValueError('''Pad_token_id must be set in the configuration of the model, in order to pad tensors'''      )

										snake_case__       :					str										=					pad_token_id * torch.ones(
										    (tensor.shape[0], max_length)      ,dtype=tensor.dtype      ,device=tensor.device      )
										snake_case__       :					str										=					tensor
										return padded_tensor








"""simple docstring"""





import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
							import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


_lowercase	      =    get_logger(__name__)

def 		lowerCAmelCase							( lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase = None      ,			):
						'''simple docstring'''


						for i, state in enumerate(lowerCamelCase		):
												snake_case__       :					List[Any]										=					f'''{MODEL_NAME}.bin''' if i == 0 else f'''{MODEL_NAME}_{i}.bin'''
												snake_case__       :					Tuple										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												save(lowerCamelCase      ,			lowerCamelCase		)
												logger.info(f'''Model weights saved in {output_model_file}'''		)
						# Optimizer states
						for i, opt in enumerate(lowerCamelCase		):
												snake_case__       :					str										=					opt.state_dict()
												snake_case__       :					Optional[Any]										=					f'''{OPTIMIZER_NAME}.bin''' if i == 0 else f'''{OPTIMIZER_NAME}_{i}.bin'''
												snake_case__       :					str										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												save(lowerCamelCase      ,			lowerCamelCase		)
												logger.info(f'''Optimizer state saved in {output_optimizer_file}'''		)
						# Scheduler states
						for i, scheduler in enumerate(lowerCamelCase		):
												snake_case__       :					int										=					scheduler.state_dict()
												snake_case__       :					List[Any]										=					f'''{SCHEDULER_NAME}.bin''' if i == 0 else f'''{SCHEDULER_NAME}_{i}.bin'''
												snake_case__       :					int										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												save(lowerCamelCase      ,			lowerCamelCase		)
												logger.info(f'''Scheduler state saved in {output_scheduler_file}'''		)
						# GradScaler state
						if scaler is not None:
												snake_case__       :					Tuple										=					scaler.state_dict()
												snake_case__       :					Tuple										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												torch.save(lowerCamelCase      ,			lowerCamelCase		)
												logger.info(f'''Gradient scaler state saved in {output_scaler_file}'''		)
						# Random number generator states
						snake_case__       :					Dict										=					{}
						snake_case__       :					Dict										=					f'''{RNG_STATE_NAME}_{process_index}.pkl'''
						snake_case__       :					Dict										=					random.getstate()
						snake_case__       :					List[str]										=					np.random.get_state()
						snake_case__       :					str										=					torch.get_rng_state()
						if is_xpu_available():
												snake_case__       :					List[Any]										=					torch.xpu.get_rng_state_all()
						else:
												snake_case__       :					Union[str, Any]										=					torch.cuda.get_rng_state_all()
						if is_tpu_available():
												snake_case__       :					int										=					xm.get_rng_state()
						snake_case__       :					int										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
						torch.save(lowerCamelCase      ,			lowerCamelCase		)
						logger.info(f'''Random states saved in {output_states_file}'''		)
						return output_dir

def 		lowerCAmelCase							( lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase=None      ,			lowerCamelCase=None      ,			**lowerCamelCase      ,			):
						'''simple docstring'''


						if map_location not in [None, "cpu", "on_device"]:
												raise TypeError(
												    '''Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`'''		)
						if map_location is None:
												snake_case__       :					str										=					'''cpu'''
						elif map_location == "on_device":
												snake_case__       :					List[Any]										=					PartialState().device
						# Model states
						for i, model in enumerate(lowerCamelCase		):
												snake_case__       :					Optional[int]										=					f'''{MODEL_NAME}.bin''' if i == 0 else f'''{MODEL_NAME}_{i}.bin'''
												snake_case__       :					int										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												models[i].load_state_dict(torch.load(lowerCamelCase      ,			map_location=lowerCamelCase		)      ,			**lowerCamelCase		)
						logger.info('''All model weights loaded successfully'''		)

						# Optimizer states
						for i, opt in enumerate(lowerCamelCase		):
												snake_case__       :					Any										=					f'''{OPTIMIZER_NAME}.bin''' if i == 0 else f'''{OPTIMIZER_NAME}_{i}.bin'''
												snake_case__       :					Optional[Any]										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												snake_case__       :					Any										=					torch.load(lowerCamelCase      ,			map_location=lowerCamelCase		)
												optimizers[i].load_state_dict(lowerCamelCase		)
						logger.info('''All optimizer states loaded successfully'''		)

						# Scheduler states
						for i, scheduler in enumerate(lowerCamelCase		):
												snake_case__       :					Optional[int]										=					f'''{SCHEDULER_NAME}.bin''' if i == 0 else f'''{SCHEDULER_NAME}_{i}.bin'''
												snake_case__       :					Optional[Any]										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												scheduler.load_state_dict(torch.load(lowerCamelCase		)		)
						logger.info('''All scheduler states loaded successfully'''		)

						# GradScaler state
						if scaler is not None:
												snake_case__       :					Optional[Any]										=					os.path.join(lowerCamelCase      ,			lowerCamelCase		)
												scaler.load_state_dict(torch.load(lowerCamelCase		)		)
												logger.info('''GradScaler state loaded successfully'''		)

						# Random states
						try:
												snake_case__       :					Optional[int]										=					torch.load(os.path.join(lowerCamelCase      ,			f'''{RNG_STATE_NAME}_{process_index}.pkl'''		)		)
												random.setstate(states['''random_state''']		)
												np.random.set_state(states['''numpy_random_seed''']		)
												torch.set_rng_state(states['''torch_manual_seed''']		)
												if is_xpu_available():
																		torch.xpu.set_rng_state_all(states['''torch_xpu_manual_seed''']		)
												else:
																		torch.cuda.set_rng_state_all(states['''torch_cuda_manual_seed''']		)
												if is_tpu_available():
																		xm.set_rng_state(states['''xm_seed''']		)
												logger.info('''All random states loaded successfully'''		)
						except Exception:
												logger.info('''Could not load random states'''		)

def 		lowerCAmelCase							( lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase = 0		):
						'''simple docstring'''


						snake_case__       :					Union[str, Any]										=					Path(lowerCamelCase		) / f'''custom_checkpoint_{index}.pkl'''
						logger.info(f'''Saving the state of {get_pretty_name(lowerCamelCase		)} to {save_location}'''		)
						torch.save(obj.state_dict()      ,			lowerCamelCase		)

def 		lowerCAmelCase							( lowerCamelCase      ,			lowerCamelCase      ,			lowerCamelCase = 0		):
						'''simple docstring'''


						snake_case__       :					Any										=					f'''{path}/custom_checkpoint_{index}.pkl'''
						logger.info(f'''Loading the state of {get_pretty_name(lowerCamelCase		)} from {load_location}'''		)
						obj.load_state_dict(torch.load(lowerCamelCase      ,			map_location='''cpu'''		)		)


