

import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
    import torch

UpperCamelCase__		   =    logging.get_logger(__name__)







def 	lowerCamelCase__			(   ):


       """simple docstring"""




       return importlib.util.find_spec("""deepspeed"""		) is not None


if is_accelerate_available() and is_deepspeed_available():
    from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
    # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
    # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
    from builtins import object as DeepSpeedConfig

class 						__snake_case						(							snake_case__     ):
       """simple docstring"""







       def __init__(		self			,   _UpperCamelCase      )			->       Optional[int]:






              """simple docstring"""


              set_hf_deepspeed_config(self      )
              dep_version_check("""accelerate"""      )
              dep_version_check("""deepspeed"""      )
              super().__init__(_UpperCamelCase      )






class 						__snake_case						(							snake_case__     ):
       """simple docstring"""







       def __init__(		self			,   _UpperCamelCase      )			->       Tuple:






              """simple docstring"""


              super().__init__(_UpperCamelCase      )
              __snake_case					  =       None
              __snake_case					  =       []


       def  a		(		self      )			->       str:






              """simple docstring"""


              if self._dtype is None:
                     raise ValueError("""trainer_config_process() wasn't called yet to tell dtype"""      )
              return self._dtype


       def  a		(		self			,   _UpperCamelCase      )			->       Dict:






              """simple docstring"""


              __snake_case					  =       self.get_value(_UpperCamelCase      )
              if val is None:
                     return False
              else:
                     return val == "auto"


       def  a		(		self			,   _UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase=None			,   _UpperCamelCase=True      )			->       str:






              """simple docstring"""


              __snake_case  ,						__snake_case					  =       self.find_config_node(_UpperCamelCase      )
              if config is None:
                     return

              if config.get(_UpperCamelCase      ) == "auto":
                     __snake_case					  =       hf_val
                     return

              if not must_match:
                     return

              __snake_case					  =       config.get(_UpperCamelCase      )
              if ds_val is not None and ds_val != hf_val:
                     self.mismatches.append(F'- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}'      )

       __SCREAMING_SNAKE_CASE													=       partialmethod(snake_case__ ,  must_match=snake_case__     )


       def  a		(		self			,   _UpperCamelCase      )			->       Any:






              """simple docstring"""


              __snake_case					  =       args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
              self.fill_match(
                  """train_micro_batch_size_per_gpu"""			,   args.per_device_train_batch_size			,   """per_device_train_batch_size"""      )
              self.fill_match("""gradient_accumulation_steps"""			,   args.gradient_accumulation_steps			,   """gradient_accumulation_steps"""      )
              self.fill_match("""train_batch_size"""			,   _UpperCamelCase			,   """train_batch_size (calculated)"""      )
              self.fill_match("""gradient_clipping"""			,   args.max_grad_norm			,   """max_grad_norm"""      )

              self.fill_match("""optimizer.params.lr"""			,   args.learning_rate			,   """learning_rate"""      )
              self.fill_match("""optimizer.params.betas"""			,   [args.adam_betaa, args.adam_betaa]			,   """adam_beta1+adam_beta2"""      )
              self.fill_match("""optimizer.params.eps"""			,   args.adam_epsilon			,   """adam_epsilon"""      )
              self.fill_match("""optimizer.params.weight_decay"""			,   args.weight_decay			,   """weight_decay"""      )

              self.fill_only("""scheduler.params.warmup_min_lr"""			,   0      )  # not a trainer arg
              self.fill_match("""scheduler.params.warmup_max_lr"""			,   args.learning_rate			,   """learning_rate"""      )
              # total_num_steps - will get set in trainer_config_finalize

              # fp16
              if args.fpaa or args.fpaa_full_eval:
                     __snake_case					  =       """apex""" if args.fpaa_backend == """apex""" else """amp"""
              else:
                     __snake_case					  =       None

              if args.save_on_each_node:
                     # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
                     __snake_case					  =       self.config.get("""checkpoint"""			,   {}      )
                     __snake_case					  =       args.save_on_each_node

              # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
              # any here unless the user did the work
              self.fill_match(
                  """fp16.enabled"""			,   ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == """amp""")			,   """fp16|fp16_full_eval+fp16_backend(amp)"""			,   )

              # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
              # ZeRO features
              self.fill_match("""amp.enabled"""			,   fpaa_backend == """apex"""			,   """fp16+fp16_backend(apex)"""      )
              self.fill_match("""amp.opt_level"""			,   args.fpaa_opt_level			,   """fp16_opt_level"""      )

              self.fill_match("""bf16.enabled"""			,   (args.bfaa or args.bfaa_full_eval)			,   """bf16|bf16_full_eval"""      )

              # deepspeed's default mode is fp16 unless there is a config that says differently
              if self.is_true("""bf16.enabled"""      ):
                     __snake_case					  =       torch.bfloataa
              elif self.is_false("""fp16.enabled"""      ):
                     __snake_case					  =       torch.floataa
              else:
                     __snake_case					  =       torch.floataa






       def  a		(		self			,   _UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase      )			->       Tuple:






              """simple docstring"""


              __snake_case					  =       [
                  """zero_optimization.reduce_bucket_size""",
                  """zero_optimization.stage3_prefetch_bucket_size""",
                  """zero_optimization.stage3_param_persistence_threshold""",
              ]
              __snake_case					  =       [x for x in hidden_size_based_keys if self.is_auto(_UpperCamelCase      )]

              if len(_UpperCamelCase      ) > 0:
                     if hasattr(model.config			,   """hidden_size"""      ):
                            __snake_case					  =       model.config.hidden_size
                     elif hasattr(model.config			,   """hidden_sizes"""      ):
                            # if there are many hidden sizes pick the largest one
                            __snake_case					  =       max(model.config.hidden_sizes      )
                     else:
                            raise ValueError(
                                """The model's config file has neither `hidden_size` nor `hidden_sizes` entry, """
                                """therefore it's not possible to automatically fill out the following `auto` entries """
                                F'in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '
                                """`auto` values for these keys with an integer value of your choice."""      )

                     self.fill_only("""zero_optimization.reduce_bucket_size"""			,   hidden_size * hidden_size      )
                     if self.is_zeroa():
                            # automatically assign the optimal config values based on model config
                            self.fill_only("""zero_optimization.stage3_prefetch_bucket_size"""			,   0.9 * hidden_size * hidden_size      )
                            self.fill_only("""zero_optimization.stage3_param_persistence_threshold"""			,   10 * hidden_size      )

        # scheduler
              self.fill_match("""scheduler.params.total_num_steps"""			,   _UpperCamelCase			,   """num_training_steps (calculated)"""      )
              self.fill_match("""scheduler.params.warmup_num_steps"""			,   args.get_warmup_steps(_UpperCamelCase      )			,   """warmup_steps"""      )

              if len(self.mismatches      ) > 0:
                     __snake_case					  =       """\n""".join(self.mismatches      )
                     raise ValueError(
                         """Please correct the following DeepSpeed config values that mismatch TrainingArguments"""
                         F' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.'      )






UpperCamelCase__		   =    None







def 	lowerCamelCase__			(   __A						:Tuple		):


       """simple docstring"""




       global _hf_deepspeed_config_weak_ref
       # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
       __snake_case					  =       weakref.ref(__A		)







def 	lowerCamelCase__			(   ):


       """simple docstring"""




       global _hf_deepspeed_config_weak_ref
       __snake_case					  =       None







def 	lowerCamelCase__			(   ):


       """simple docstring"""




       if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
              return _hf_deepspeed_config_weak_ref().is_zeroa()
       else:
              return False







def 	lowerCamelCase__			(   ):


       """simple docstring"""




       if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
              return _hf_deepspeed_config_weak_ref().config
       else:
              return None







def 	lowerCamelCase__			(   __A						:int  ,__A						:Union[str, Any]  ,__A						:Tuple  ,__A						:Union[str, Any]  ,__A						:List[str]		):


       """simple docstring"""




       from accelerate.utils import DummyOptim, DummyScheduler

       __snake_case					  =       hf_deepspeed_config.config

       # Optimizer + Scheduler
       # Currently supported combos:
       # 1. DS scheduler + DS optimizer: Yes
       # 2. HF scheduler + HF optimizer: Yes
       # 3. DS scheduler + HF optimizer: Yes
       # 4. HF scheduler + DS optimizer: No
       #
       # Unless Offload is enabled in which case it's:
       # 1. DS scheduler + DS optimizer: Yes
       # 2. HF scheduler + HF optimizer: Mostly*
       # 3. DS scheduler + HF optimizer: Mostly*
       # 4. HF scheduler + DS optimizer: No
       #
       # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

       __snake_case					  =       None
       if "optimizer" in config:
              if args.adafactor:
                     raise ValueError(
                         """--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. """
                         """Only one optimizer can be configured."""		)
              __snake_case					  =       DummyOptim(params=__A		)
       else:
              if hf_deepspeed_config.is_offload():
                     logger.info(
                         """Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"""
                         """ custom optimizer has both CPU and GPU implementation (except LAMB)"""		)

              # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
              # But trainer uses AdamW by default.
              __snake_case					  =       trainer.create_optimizer()
              # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
              __snake_case					  =       True

       __snake_case					  =       None
       if "scheduler" in config:
              __snake_case					  =       DummyScheduler(__A		)
       else:
              if isinstance(__A  ,__A		):
                     raise ValueError(
                         """Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. """
                         """Please configure a scheduler in the DeepSpeed config."""		)
              __snake_case					  =       trainer.create_scheduler(num_training_steps=__A  ,optimizer=__A		)

       return optimizer, lr_scheduler







def 	lowerCamelCase__			(   __A						:Tuple  ,__A						:Optional[Any]  ,__A						:Dict=False		):


       """simple docstring"""




       from deepspeed.utils import logger as ds_logger

       __snake_case					  =       trainer.model
       __snake_case					  =       trainer.args

       __snake_case					  =       trainer.accelerator.state.deepspeed_plugin.hf_ds_config

       # resume config update - some bits like `model` and `num_training_steps` only become available during train
       hf_deepspeed_config.trainer_config_finalize(__A  ,__A  ,__A		)

       # set the Deepspeed log level consistent with the Trainer
       ds_logger.setLevel(args.get_process_log_level()		)

       if inference:
              # only Z3 makes sense for the inference
              if not hf_deepspeed_config.is_zeroa():
                     raise ValueError("""ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"""		)

              # in case the training config is re-used for inference
              hf_deepspeed_config.del_config_sub_tree("""optimizer"""		)
              hf_deepspeed_config.del_config_sub_tree("""lr_scheduler"""		)
              __snake_case  ,						__snake_case					  =       None, None
              __snake_case					  =       None
       else:
              __snake_case					  =       None  # important for when deepspeed_init is used as re-init
              __snake_case					  =       list(filter(lambda __A		: p.requires_grad  ,model.parameters()		)		)
              __snake_case  ,						__snake_case					  =       deepspeed_optim_sched(
                  __A  ,__A  ,__A  ,__A  ,__A		)

       # keep for quick debug:
       # from pprint import pprint; pprint(config)

       return optimizer, lr_scheduler







def 	lowerCamelCase__			(   __A						:Tuple  ,__A						:Union[str, Any]		):


       """simple docstring"""




       import glob

       __snake_case					  =       sorted(glob.glob(F'{checkpoint_path}/global_step*'		)		)

       if len(__A		) > 0:
              logger.info(F'Attempting to resume from {checkpoint_path}'		)
              # this magically updates self.optimizer and self.lr_scheduler
              __snake_case  ,						__snake_case					  =       deepspeed_engine.load_checkpoint(
                  __A  ,load_optimizer_states=__A  ,load_lr_scheduler_states=__A		)
              if load_path is None:
                     raise ValueError(F'[deepspeed] failed to resume from checkpoint {checkpoint_path}'		)
       else:
              raise ValueError(F'Can\'t find a valid checkpoint at {checkpoint_path}'		)




import datasets

from .evaluate import (
    apply_no_ans_threshold,
    find_all_best_thresh,
    get_raw_scores,
    make_eval_dict,
    make_qid_to_has_ans,
    merge_eval,
)


UpperCamelCase__		   =    '''\
@inproceedings{Rajpurkar2016SQuAD10,
  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},
  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},
  booktitle={EMNLP},
  year={2016}
}
'''

UpperCamelCase__		   =    '''
This metric wrap the official scoring script for version 2 of the Stanford Question
Answering Dataset (SQuAD).

Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by
crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,
from the corresponding reading passage, or the question might be unanswerable.

SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions
written adversarially by crowdworkers to look similar to answerable ones.
To do well on SQuAD2.0, systems must not only answer questions when possible, but also
determine when no answer is supported by the paragraph and abstain from answering.
'''

UpperCamelCase__		   =    '''
Computes SQuAD v2 scores (F1 and EM).
Args:
    predictions: List of triple for question-answers to score with the following elements:
        - the question-answer \'id\' field as given in the references (see below)
        - the text of the answer
        - the probability that the question has no answer
    references: List of question-answers dictionaries with the following key-values:
            - \'id\': id of the question-answer pair (see above),
            - \'answers\': a list of Dict {\'text\': text of the answer as a string}
    no_answer_threshold: float
        Probability threshold to decide that a question has no answer.
Returns:
    \'exact\': Exact match (the normalized answer exactly match the gold answer)
    \'f1\': The F-score of predicted tokens versus the gold answer
    \'total\': Number of score considered
    \'HasAns_exact\': Exact match (the normalized answer exactly match the gold answer)
    \'HasAns_f1\': The F-score of predicted tokens versus the gold answer
    \'HasAns_total\': Number of score considered
    \'NoAns_exact\': Exact match (the normalized answer exactly match the gold answer)
    \'NoAns_f1\': The F-score of predicted tokens versus the gold answer
    \'NoAns_total\': Number of score considered
    \'best_exact\': Best exact match (with varying threshold)
    \'best_exact_thresh\': No-answer probability threshold associated to the best exact match
    \'best_f1\': Best F1 (with varying threshold)
    \'best_f1_thresh\': No-answer probability threshold associated to the best F1
Examples:

    >>> predictions = [{\'prediction_text\': \'1976\', \'id\': \'56e10a3be3433e1400422b22\', \'no_answer_probability\': 0.}]
    >>> references = [{\'answers\': {\'answer_start\': [97], \'text\': [\'1976\']}, \'id\': \'56e10a3be3433e1400422b22\'}]
    >>> squad_v2_metric = datasets.load_metric("squad_v2")
    >>> results = squad_v2_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact\': 100.0, \'f1\': 100.0, \'total\': 1, \'HasAns_exact\': 100.0, \'HasAns_f1\': 100.0, \'HasAns_total\': 1, \'best_exact\': 100.0, \'best_exact_thresh\': 0.0, \'best_f1\': 100.0, \'best_f1_thresh\': 0.0}
'''

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,  _KWARGS_DESCRIPTION     )
class 						__snake_case						(							datasets.Metric     ):
       """simple docstring"""







       def  a		(		self      )			->       Tuple:






              """simple docstring"""


              return datasets.MetricInfo(
                  description=_DESCRIPTION			,   citation=_CITATION			,   inputs_description=_KWARGS_DESCRIPTION			,   features=datasets.Features(
                      {
                          """predictions""": {
                              """id""": datasets.Value("""string"""      ),
                              """prediction_text""": datasets.Value("""string"""      ),
                              """no_answer_probability""": datasets.Value("""float32"""      ),
                          },
                          """references""": {
                              """id""": datasets.Value("""string"""      ),
                              """answers""": datasets.features.Sequence(
                                  {"""text""": datasets.Value("""string"""      ), """answer_start""": datasets.Value("""int32"""      )}      ),
                          },
                      }      )			,   codebase_urls=["""https://rajpurkar.github.io/SQuAD-explorer/"""]			,   reference_urls=["""https://rajpurkar.github.io/SQuAD-explorer/"""]			,   )






       def  a		(		self			,   _UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase=1.0      )			->       Dict:






              """simple docstring"""


              __snake_case					  =       {p["""id"""]: p["""no_answer_probability"""] for p in predictions}
              __snake_case					  =       [{"""paragraphs""": [{"""qas""": references}]}]
              __snake_case					  =       {p["""id"""]: p["""prediction_text"""] for p in predictions}

              __snake_case					  =       make_qid_to_has_ans(_UpperCamelCase      )  # maps qid to True/False
              __snake_case					  =       [k for k, v in qid_to_has_ans.items() if v]
              __snake_case					  =       [k for k, v in qid_to_has_ans.items() if not v]

              __snake_case  ,						__snake_case					  =       get_raw_scores(_UpperCamelCase			,   _UpperCamelCase      )
              __snake_case					  =       apply_no_ans_threshold(_UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase      )
              __snake_case					  =       apply_no_ans_threshold(_UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase      )
              __snake_case					  =       make_eval_dict(_UpperCamelCase			,   _UpperCamelCase      )

              if has_ans_qids:
                     __snake_case					  =       make_eval_dict(_UpperCamelCase			,   _UpperCamelCase			,   qid_list=_UpperCamelCase      )
                     merge_eval(_UpperCamelCase			,   _UpperCamelCase			,   """HasAns"""      )
              if no_ans_qids:
                     __snake_case					  =       make_eval_dict(_UpperCamelCase			,   _UpperCamelCase			,   qid_list=_UpperCamelCase      )
                     merge_eval(_UpperCamelCase			,   _UpperCamelCase			,   """NoAns"""      )
              find_all_best_thresh(_UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase      )
              return dict(_UpperCamelCase      )

