





'''simple docstring'''


import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


A_				     =							16
A_				     =							32




def       A							(      _UpperCAmelCase   :				Optional[Any]					)  ->				Dict:


  '''simple docstring'''




  return int(x / 2**2_0					)


class 		UpperCamelCase__     :
      '''simple docstring'''






      def __enter__(     self )				->  Tuple:
        gc.collect()
        torch.cuda.empty_cache()
        torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
        __lowerCAmelCase		:    Optional[int]			     =  torch.cuda.memory_allocated()
        __lowerCAmelCase		:    Tuple			     =  psutil.Process()

        __lowerCAmelCase		:    List[str]			     =  self.cpu_mem_used()
        __lowerCAmelCase		:    int			     =  True
        __lowerCAmelCase		:    List[str]			     =  threading.Thread(target=self.peak_monitor_func )
        __lowerCAmelCase		:    Optional[int]			     =  True
        peak_monitor_thread.start()
        return self




      def 	snake_case  (     self )				->  Union[str, Any]:
        return self.process.memory_info().rss




      def 	snake_case  (     self )				->  Dict:
        __lowerCAmelCase		:    Optional[int]			     =  -1

        while True:
          __lowerCAmelCase		:    Tuple			     =  max(self.cpu_mem_used()     , self.cpu_peak )

          # can't sleep or will not catch the peak right (this comment is here on purpose)
          # time.sleep(0.001) # 1msec

          if not self.peak_monitoring:
            break







      def __exit__(     self     , *SCREAMING_SNAKE_CASE )				->  Tuple:
        __lowerCAmelCase		:    Tuple			     =  False

        gc.collect()
        torch.cuda.empty_cache()
        __lowerCAmelCase		:    Union[str, Any]			     =  torch.cuda.memory_allocated()
        __lowerCAmelCase		:    Tuple			     =  torch.cuda.max_memory_allocated()
        __lowerCAmelCase		:    str			     =  bamb(self.end - self.begin )
        __lowerCAmelCase		:    Optional[int]			     =  bamb(self.peak - self.begin )

        __lowerCAmelCase		:    str			     =  self.cpu_mem_used()
        __lowerCAmelCase		:    List[Any]			     =  bamb(self.cpu_end - self.cpu_begin )
        __lowerCAmelCase		:    Any			     =  bamb(self.cpu_peak - self.cpu_begin )
        # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
    from accelerate.test_utils.training import mocked_dataloaders

    A_				     =							mocked_dataloaders  # noqa: F811




def       A							(      _UpperCAmelCase   :				List[Any]						,_UpperCAmelCase   :				List[str]					)  ->				Any:


  '''simple docstring'''




  if os.environ.get('TESTING_MOCKED_DATALOADERS'						,_lowerCAmelCase					) == "1":
    __lowerCAmelCase		:    Union[str, Any]			     =  2

  # New Code #
  # Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
  __lowerCAmelCase		:    Tuple			     =  FullyShardedDataParallelPlugin(
      state_dict_config=FullStateDictConfig(offload_to_cpu=_lowerCAmelCase						,ranka_only=_lowerCAmelCase					)						,optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=_lowerCAmelCase						,ranka_only=_lowerCAmelCase					)						,)

  # Initialize accelerator
  if args.with_tracking:
    __lowerCAmelCase		:    Tuple			     =  Accelerator(
        cpu=args.cpu						,mixed_precision=args.mixed_precision						,log_with='wandb'						,project_dir=args.logging_dir						,fsdp_plugin=_lowerCAmelCase						,)
  else:
    __lowerCAmelCase		:    List[Any]			     =  Accelerator(fsdp_plugin=_lowerCAmelCase					)
  accelerator.print(accelerator.distributed_type					)

  if hasattr(args.checkpointing_steps						,'isdigit'					):
    if args.checkpointing_steps == "epoch":
      __lowerCAmelCase		:    Union[str, Any]			     =  args.checkpointing_steps
    elif args.checkpointing_steps.isdigit():
      __lowerCAmelCase		:    Any			     =  int(args.checkpointing_steps					)
    else:
      raise ValueError(
          F"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""					)
  else:
    __lowerCAmelCase		:    Optional[int]			     =  None
  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  __lowerCAmelCase		:    Union[str, Any]			     =  config['lr']
  __lowerCAmelCase		:    Dict			     =  int(config['num_epochs']					)
  __lowerCAmelCase		:    int			     =  int(config['seed']					)
  __lowerCAmelCase		:    Union[str, Any]			     =  int(config['batch_size']					)

  # We need to initialize the trackers we use, and also store our configuration
  if args.with_tracking:
    __lowerCAmelCase		:    Any			     =  vars(_lowerCAmelCase					)
    accelerator.init_trackers('fsdp_glue_no_trainer'						,_lowerCAmelCase					)

  __lowerCAmelCase		:    List[str]			     =  AutoTokenizer.from_pretrained(args.model_name_or_path					)
  __lowerCAmelCase		:    Union[str, Any]			     =  load_dataset('glue'						,'mrpc'					)
  __lowerCAmelCase		:    Dict			     =  evaluate.load('glue'						,'mrpc'					)

  def tokenize_function(_UpperCAmelCase   :				Optional[int]					):
    # max_length=None => use the model max length (it's actually the default)
    __lowerCAmelCase		:    Tuple			     =  tokenizer(examples['sentence1']						,examples['sentence2']						,truncation=_lowerCAmelCase						,max_length=_lowerCAmelCase					)
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  # starting with the main process first:
  with accelerator.main_process_first():
    __lowerCAmelCase		:    str			     =  datasets.map(
        _lowerCAmelCase						,batched=_lowerCAmelCase						,remove_columns=['idx', 'sentence1', 'sentence2']						,)

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  __lowerCAmelCase		:    Optional[int]			     =  tokenized_datasets.rename_column('label'						,'labels'					)

  # If the batch size is too big we use gradient accumulation
  __lowerCAmelCase		:    int			     =  1
  if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
    __lowerCAmelCase		:    int			     =  batch_size // MAX_GPU_BATCH_SIZE
    __lowerCAmelCase		:    Union[str, Any]			     =  MAX_GPU_BATCH_SIZE

  def collate_fn(_UpperCAmelCase   :				Optional[Any]					):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    __lowerCAmelCase		:    Any			     =  1_2_8 if accelerator.distributed_type == DistributedType.TPU else None
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
      __lowerCAmelCase		:    List[str]			     =  1_6
    elif accelerator.mixed_precision != "no":
      __lowerCAmelCase		:    List[Any]			     =  8
    else:
      __lowerCAmelCase		:    Union[str, Any]			     =  None

    return tokenizer.pad(
        _lowerCAmelCase						,padding='longest'						,max_length=_lowerCAmelCase						,pad_to_multiple_of=_lowerCAmelCase						,return_tensors='pt'						,)

  # Instantiate dataloaders.
  __lowerCAmelCase		:    str			     =  DataLoader(
      tokenized_datasets['train']						,shuffle=_lowerCAmelCase						,collate_fn=_lowerCAmelCase						,batch_size=_lowerCAmelCase					)
  __lowerCAmelCase		:    Any			     =  DataLoader(
      tokenized_datasets['validation']						,shuffle=_lowerCAmelCase						,collate_fn=_lowerCAmelCase						,batch_size=_lowerCAmelCase					)

  set_seed(_lowerCAmelCase					)

  # Instantiate the model (we build the model here so that the seed also control new weights initialization)
  __lowerCAmelCase		:    int			     =  AutoModelForSequenceClassification.from_pretrained(
      args.model_name_or_path						,return_dict=_lowerCAmelCase						,low_cpu_mem_usage=_lowerCAmelCase					)

  # New Code #
  # For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
  __lowerCAmelCase		:    List[str]			     =  accelerator.prepare(_lowerCAmelCase					)
  accelerator.print(_lowerCAmelCase					)

  # Instantiate optimizer
  # New Code #
  # For FSDP feature, at present it doesn't support multiple parameter groups,
  # so we need to create a single parameter group for the whole model
  __lowerCAmelCase		:    Optional[int]			     =  torch.optim.AdamW(params=model.parameters()						,lr=_lowerCAmelCase						,weight_decay=2e-4					)

  # Instantiate scheduler
  __lowerCAmelCase		:    Optional[Any]			     =  get_linear_schedule_with_warmup(
      optimizer=_lowerCAmelCase						,num_warmup_steps=1_0						,num_training_steps=(len(_lowerCAmelCase					) * num_epochs) // gradient_accumulation_steps						,)

  # New Code #
  # For FSDP feature, prepare everything except the model as we have already prepared the model
  # before creating the optimizer
  # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  # prepare method.
  __lowerCAmelCase     ,					__lowerCAmelCase     ,					__lowerCAmelCase     ,					__lowerCAmelCase		:    Tuple			     =  accelerator.prepare(
      _lowerCAmelCase						,_lowerCAmelCase						,_lowerCAmelCase						,_lowerCAmelCase					)

  __lowerCAmelCase		:    List[str]			     =  0

  # Potentially load in the weights and states from a previous save
  if args.resume_from_checkpoint:
    if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
      accelerator.print(F"""Resumed from checkpoint: {args.resume_from_checkpoint}"""					)
      accelerator.load_state(args.resume_from_checkpoint					)
      __lowerCAmelCase		:    str			     =  os.path.basename(args.resume_from_checkpoint					)
    else:
      # Get the most recent checkpoint
      __lowerCAmelCase		:    Tuple			     =  [f.name for f in os.scandir(os.getcwd()					) if f.is_dir()]
      dirs.sort(key=os.path.getctime					)
      __lowerCAmelCase		:    Optional[Any]			     =  dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
    # Extract `epoch_{i}` or `step_{i}`
    __lowerCAmelCase		:    str			     =  os.path.splitext(_lowerCAmelCase					)[0]

    if "epoch" in training_difference:
      num_epochs -= int(training_difference.replace('epoch_'						,''					)					)
      __lowerCAmelCase		:    List[str]			     =  None
    else:
      __lowerCAmelCase		:    Optional[Any]			     =  int(training_difference.replace('step_'						,''					)					)
      num_epochs -= resume_step // len(_lowerCAmelCase					)
      # If resuming by step, we also need to know exactly how far into the DataLoader we went
      __lowerCAmelCase		:    str			     =  (num_epochs * len(_lowerCAmelCase					)) - resume_step

    # Now we train the model
  for epoch in range(_lowerCAmelCase					):
    # New Code #
    # context manager to track the peak memory usage during the training epoch
    with TorchTracemalloc() as tracemalloc:
      model.train()
      if args.with_tracking:
        __lowerCAmelCase		:    Any			     =  0
      for step, batch in enumerate(_lowerCAmelCase					):
        # We need to skip steps until we reach the resumed step
        if args.resume_from_checkpoint and epoch == 0:
          if resume_step is not None and step < resume_step:
            pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
        batch.to(accelerator.device					)
        __lowerCAmelCase		:    Optional[Any]			     =  model(**_lowerCAmelCase					)
        __lowerCAmelCase		:    List[str]			     =  outputs.loss
        __lowerCAmelCase		:    Optional[int]			     =  loss / gradient_accumulation_steps
        # We keep track of the loss at each epoch
        if args.with_tracking:
          total_loss += loss.detach().float()
        accelerator.backward(_lowerCAmelCase					)
        if step % gradient_accumulation_steps == 0:
          optimizer.step()
          lr_scheduler.step()
          optimizer.zero_grad()
          # accelerator.print(lr_scheduler.get_lr())

        overall_step += 1

        if isinstance(_lowerCAmelCase						,_lowerCAmelCase					):
          __lowerCAmelCase		:    Tuple			     =  F"""step_{overall_step}"""
          if overall_step % checkpointing_steps == 0:
            if args.output_dir is not None:
              __lowerCAmelCase		:    List[str]			     =  os.path.join(args.output_dir						,_lowerCAmelCase					)
            accelerator.save_state(_lowerCAmelCase					)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
    accelerator.print('Memory before entering the train : {}'.format(bamb(tracemalloc.begin					)					)					)
    accelerator.print('Memory consumed at the end of the train (end-begin): {}'.format(tracemalloc.used					)					)
    accelerator.print('Peak Memory consumed during the train (max-begin): {}'.format(tracemalloc.peaked					)					)
    accelerator.print(
        'Total Peak Memory consumed during the train (max): {}'.format(
            tracemalloc.peaked + bamb(tracemalloc.begin					)					)					)
    # Logging the peak memory usage of the GPU to the tracker
    if args.with_tracking:
      accelerator.log(
          {
              'train_total_peak_memory': tracemalloc.peaked + bamb(tracemalloc.begin					),
          }						,step=_lowerCAmelCase						,)

    # New Code #
    # context manager to track the peak memory usage during the evaluation
    with TorchTracemalloc() as tracemalloc:
      model.eval()
      for step, batch in enumerate(_lowerCAmelCase					):
        # We could avoid this line since we set the accelerator with `device_placement=True`.
        batch.to(accelerator.device					)
        with torch.no_grad():
          __lowerCAmelCase		:    str			     =  model(**_lowerCAmelCase					)
        __lowerCAmelCase		:    Optional[int]			     =  outputs.logits.argmax(dim=-1					)
        __lowerCAmelCase     ,					__lowerCAmelCase		:    Optional[int]			     =  accelerator.gather_for_metrics((predictions, batch['labels'])					)
        metric.add_batch(
            predictions=_lowerCAmelCase						,references=_lowerCAmelCase						,)

      __lowerCAmelCase		:    Optional[int]			     =  metric.compute()
      # Use accelerator.print to print only on the main process.
      accelerator.print(F"""epoch {epoch}:"""						,_lowerCAmelCase					)
      if args.with_tracking:
        accelerator.log(
            {
                'accuracy': eval_metric['accuracy'],
                'f1': eval_metric['f1'],
                'train_loss': total_loss.item() / len(_lowerCAmelCase					),
            }						,step=_lowerCAmelCase						,)

      if checkpointing_steps == "epoch":
        __lowerCAmelCase		:    int			     =  F"""epoch_{epoch}"""
        if args.output_dir is not None:
          __lowerCAmelCase		:    List[str]			     =  os.path.join(args.output_dir						,_lowerCAmelCase					)
        accelerator.save_state(_lowerCAmelCase					)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
    accelerator.print('Memory before entering the eval : {}'.format(bamb(tracemalloc.begin					)					)					)
    accelerator.print('Memory consumed at the end of the eval (end-begin): {}'.format(tracemalloc.used					)					)
    accelerator.print('Peak Memory consumed during the eval (max-begin): {}'.format(tracemalloc.peaked					)					)
    accelerator.print(
        'Total Peak Memory consumed during the eval (max): {}'.format(tracemalloc.peaked + bamb(tracemalloc.begin					)					)					)
    # Logging the peak memory usage of the GPU to the tracker
    if args.with_tracking:
      accelerator.log(
          {
              'eval_total_peak_memory': tracemalloc.peaked + bamb(tracemalloc.begin					),
          }						,step=_lowerCAmelCase						,)

  if args.with_tracking:
    accelerator.end_training()




def       A							(      )  ->				Union[str, Any]:


  '''simple docstring'''




  __lowerCAmelCase		:    List[str]			     =  argparse.ArgumentParser(description='Simple example of training script.'					)
  parser.add_argument(
      '--mixed_precision'						,type=_lowerCAmelCase						,default=_lowerCAmelCase						,choices=['no', 'fp16', 'bf16', 'fp8']						,help='Whether to use mixed precision. Choose'
      'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
      'and an Nvidia Ampere GPU.'						,)
  parser.add_argument('--cpu'						,action='store_true'						,help='If passed, will train on the CPU.'					)
  parser.add_argument(
      '--checkpointing_steps'						,type=_lowerCAmelCase						,default=_lowerCAmelCase						,help='Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'						,)
  parser.add_argument(
      '--resume_from_checkpoint'						,type=_lowerCAmelCase						,default=_lowerCAmelCase						,help='If the training should continue from a checkpoint folder.'						,)
  parser.add_argument(
      '--with_tracking'						,action='store_true'						,help='Whether to load in all available experiment trackers from the environment and use them for logging.'						,)
  parser.add_argument(
      '--output_dir'						,type=_lowerCAmelCase						,default='.'						,help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'						,)
  parser.add_argument(
      '--logging_dir'						,type=_lowerCAmelCase						,default='logs'						,help='Location on where to store experiment tracking logs`'						,)
  parser.add_argument(
      '--model_name_or_path'						,type=_lowerCAmelCase						,help='Path to pretrained model or model identifier from huggingface.co/models.'						,required=_lowerCAmelCase						,)
  __lowerCAmelCase		:    Optional[Any]			     =  parser.parse_args()
  __lowerCAmelCase		:    Union[str, Any]			     =  {'lr': 2e-5, 'num_epochs': 3, 'seed': 4_2, 'batch_size': 1_6}
  training_function(_lowerCAmelCase						,_lowerCAmelCase					)


if __name__ == "__main__":
    main()










def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 9 / 5) + 32 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(float(_lowerCAmelCase) + 273.15 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 9 / 5) + 491.67 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) - 32) * 5 / 9 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(((float(_lowerCAmelCase) - 32) * 5 / 9) + 273.15 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(float(_lowerCAmelCase) + 459.67 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(float(_lowerCAmelCase) - 273.15 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(((float(_lowerCAmelCase) - 273.15) * 9 / 5) + 32 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 9 / 5) ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) - 491.67) * 5 / 9 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(float(_lowerCAmelCase) - 459.67 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 5 / 9) ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 1.25 + 273.15) ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 2.25 + 32) ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 1.25) ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 2.25 + 32 + 459.67) ,						_lowerCAmelCase)


if __name__ == "__main__":
	import doctest

	doctest.testmod()
