


'''simple docstring'''





import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only

from utils import save_json






def 	UpperCAmelCase__(							_SCREAMING_SNAKE_CASE				:   Tuple   ):






			"""simple docstring"""




			__A=  filter(lambda _SCREAMING_SNAKE_CASE   : p.requires_grad,model.parameters()   )
			__A=  sum([np.prod(p.size()   ) for p in model_parameters]   )
			return params


UpperCAmelCase__      =							logging.getLogger(__name__)
class 	a__				(				pl.Callback      ):





			'''simple docstring'''

			def 				lowerCAmelCase    (     self       : Any ,						lowerCAmelCase_       : Optional[Any] ,						lowerCAmelCase_       : List[Any]  )			->		Optional[int]:
						__A=  {F"""lr_group_{i}""": param['lr'] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups  )}
						pl_module.logger.log_metrics(lowerCAmelCase_  )
			@rank_zero_only
			def 				lowerCAmelCase    (     self       : Tuple ,						lowerCAmelCase_       : pl.Trainer ,						lowerCAmelCase_       : pl.LightningModule ,						lowerCAmelCase_       : str ,						lowerCAmelCase_       : Dict=True  )			->		None:
						logger.info(F"""***** {type_path} results at step {trainer.global_step:05d} *****"""  )
						__A=  trainer.callback_metrics
						trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ['log', 'progress_bar', 'preds']}  )
						# Log results
						__A=  Path(pl_module.hparams.output_dir  )
						if type_path == "test":
									__A=  od / 'test_results.txt'
									__A=  od / 'test_generations.txt'
						else:
									# this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
									# If people want this it will be easy enough to add back.
									__A=  od / F"""{type_path}_results/{trainer.global_step:05d}.txt"""
									__A=  od / F"""{type_path}_generations/{trainer.global_step:05d}.txt"""
									results_file.parent.mkdir(exist_ok=lowerCAmelCase_  )
									generations_file.parent.mkdir(exist_ok=lowerCAmelCase_  )
						with open(lowerCAmelCase_ ,						'a+'  ) as writer:
									for key in sorted(lowerCAmelCase_  ):
												if key in ["log", "progress_bar", "preds"]:
															continue
												__A=  metrics[key]
												if isinstance(lowerCAmelCase_ ,						torch.Tensor  ):
															__A=  val.item()
												__A=  F"""{key}: {val:.6f}\n"""
												writer.write(lowerCAmelCase_  )

						if not save_generations:
									return

						if "preds" in metrics:
									__A=  '\n'.join(metrics['preds']  )
									generations_file.open('w+'  ).write(lowerCAmelCase_  )
			@rank_zero_only
			def 				lowerCAmelCase    (     self       : Dict ,						lowerCAmelCase_       : List[str] ,						lowerCAmelCase_       : Any  )			->		str:
						try:
									__A=  pl_module.model.model.num_parameters()
						except AttributeError:
									__A=  pl_module.model.num_parameters()

						__A=  count_trainable_parameters(lowerCAmelCase_  )
						# mp stands for million parameters
						trainer.logger.log_metrics({'n_params': npars, 'mp': npars / 1E6, 'grad_mp': n_trainable_pars / 1E6}  )
			@rank_zero_only
			def 				lowerCAmelCase    (     self       : Optional[int] ,						lowerCAmelCase_       : pl.Trainer ,						lowerCAmelCase_       : pl.LightningModule  )			->		Optional[int]:
						save_json(pl_module.metrics ,						pl_module.metrics_save_path  )
						return self._write_logs(lowerCAmelCase_ ,						lowerCAmelCase_ ,						'test'  )

			@rank_zero_only
			def 				lowerCAmelCase    (     self       : Optional[int] ,						lowerCAmelCase_       : pl.Trainer ,						lowerCAmelCase_       : Dict  )			->		Optional[Any]:
						save_json(pl_module.metrics ,						pl_module.metrics_save_path  )
						# Uncommenting this will save val generations
						# return self._write_logs(trainer, pl_module, "valid")






def 	UpperCAmelCase__(							_SCREAMING_SNAKE_CASE				:   Tuple,_SCREAMING_SNAKE_CASE				:   Any,_SCREAMING_SNAKE_CASE				:   Dict=1,_SCREAMING_SNAKE_CASE				:   str=False   ):






			"""simple docstring"""




			if metric == "rouge2":
						__A=  '{val_avg_rouge2:.4f}-{step_count}'
			elif metric == "bleu":
						__A=  '{val_avg_bleu:.4f}-{step_count}'
			elif metric == "loss":
						__A=  '{val_avg_loss:.4f}-{step_count}'
			else:
						raise NotImplementedError(
						    f"""seq2seq callbacks only support rouge2, bleu and loss, got {metric}, You can make your own by adding to"""
						    ' this function.'   )

			__A=  ModelCheckpoint(
			    dirpath=_SCREAMING_SNAKE_CASE,filename=_SCREAMING_SNAKE_CASE,monitor=f"""val_{metric}""",mode='min' if 'loss' in metric else 'max',save_top_k=_SCREAMING_SNAKE_CASE,)
			return checkpoint_callback






def 	UpperCAmelCase__(							_SCREAMING_SNAKE_CASE				:   Optional[Any],_SCREAMING_SNAKE_CASE				:   str   ):






			"""simple docstring"""




			return EarlyStopping(
			    monitor=f"""val_{metric}""",mode='min' if 'loss' in metric else 'max',patience=_SCREAMING_SNAKE_CASE,verbose=_SCREAMING_SNAKE_CASE,)







'''simple docstring'''





import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


UpperCAmelCase__      =							1_6
UpperCAmelCase__      =							3_2






def 	UpperCAmelCase__(							_SCREAMING_SNAKE_CASE				:   List[str]   ):






			"""simple docstring"""




			return int(x / 2**20   )
class 	a__				:





			'''simple docstring'''

			def __enter__(     self       : str  )			->		Optional[Any]:
						gc.collect()
						torch.cuda.empty_cache()
						torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
						__A=  torch.cuda.memory_allocated()
						__A=  psutil.Process()

						__A=  self.cpu_mem_used()
						__A=  True
						__A=  threading.Thread(target=self.peak_monitor_func  )
						__A=  True
						peak_monitor_thread.start()
						return self
			def 				lowerCAmelCase    (     self       : int  )			->		List[Any]:
						return self.process.memory_info().rss
			def 				lowerCAmelCase    (     self       : str  )			->		List[Any]:
						__A=  -1

						while True:
									__A=  max(self.cpu_mem_used() ,						self.cpu_peak  )

									# can't sleep or will not catch the peak right (this comment is here on purpose)
									# time.sleep(0.001) # 1msec

									if not self.peak_monitoring:
												break

			def __exit__(     self       : List[Any] ,						*lowerCAmelCase_       : Dict  )			->		List[Any]:
						__A=  False

						gc.collect()
						torch.cuda.empty_cache()
						__A=  torch.cuda.memory_allocated()
						__A=  torch.cuda.max_memory_allocated()
						__A=  bamb(self.end - self.begin  )
						__A=  bamb(self.peak - self.begin  )

						__A=  self.cpu_mem_used()
						__A=  bamb(self.cpu_end - self.cpu_begin  )
						__A=  bamb(self.cpu_peak - self.cpu_begin  )
						# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
					from accelerate.test_utils.training import mocked_dataloaders

					UpperCAmelCase__      =							mocked_dataloaders  # noqa: F811






def 	UpperCAmelCase__(							_SCREAMING_SNAKE_CASE				:   Union[str, Any],_SCREAMING_SNAKE_CASE				:   Optional[int]   ):






			"""simple docstring"""




			if os.environ.get('TESTING_MOCKED_DATALOADERS',_SCREAMING_SNAKE_CASE   ) == "1":
						__A=  2

			# New Code #
			# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
			__A=  FullyShardedDataParallelPlugin(
			    state_dict_config=FullStateDictConfig(offload_to_cpu=_SCREAMING_SNAKE_CASE,ranka_only=_SCREAMING_SNAKE_CASE   ),optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=_SCREAMING_SNAKE_CASE,ranka_only=_SCREAMING_SNAKE_CASE   ),)

			# Initialize accelerator
			if args.with_tracking:
						__A=  Accelerator(
						    cpu=args.cpu,mixed_precision=args.mixed_precision,log_with='wandb',project_dir=args.logging_dir,fsdp_plugin=_SCREAMING_SNAKE_CASE,)
			else:
						__A=  Accelerator(fsdp_plugin=_SCREAMING_SNAKE_CASE   )
			accelerator.print(accelerator.distributed_type   )

			if hasattr(args.checkpointing_steps,'isdigit'   ):
						if args.checkpointing_steps == "epoch":
									__A=  args.checkpointing_steps
						elif args.checkpointing_steps.isdigit():
									__A=  int(args.checkpointing_steps   )
						else:
									raise ValueError(
									    f"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""   )
			else:
						__A=  None
			# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
			__A=  config['lr']
			__A=  int(config['num_epochs']   )
			__A=  int(config['seed']   )
			__A=  int(config['batch_size']   )

			# We need to initialize the trackers we use, and also store our configuration
			if args.with_tracking:
						__A=  vars(_SCREAMING_SNAKE_CASE   )
						accelerator.init_trackers('fsdp_glue_no_trainer',_SCREAMING_SNAKE_CASE   )

			__A=  AutoTokenizer.from_pretrained(args.model_name_or_path   )
			__A=  load_dataset('glue','mrpc'   )
			__A=  evaluate.load('glue','mrpc'   )

			def tokenize_function(_SCREAMING_SNAKE_CASE				:   str   ):
						# max_length=None => use the model max length (it's actually the default)
						__A=  tokenizer(examples['sentence1'],examples['sentence2'],truncation=_SCREAMING_SNAKE_CASE,max_length=_SCREAMING_SNAKE_CASE   )
						return outputs

			# Apply the method we just defined to all the examples in all the splits of the dataset
			# starting with the main process first:
			with accelerator.main_process_first():
						__A=  datasets.map(
						    _SCREAMING_SNAKE_CASE,batched=_SCREAMING_SNAKE_CASE,remove_columns=['idx', 'sentence1', 'sentence2'],)

			# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
			# transformers library
			__A=  tokenized_datasets.rename_column('label','labels'   )

			# If the batch size is too big we use gradient accumulation
			__A=  1
			if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
						__A=  batch_size // MAX_GPU_BATCH_SIZE
						__A=  MAX_GPU_BATCH_SIZE

			def collate_fn(_SCREAMING_SNAKE_CASE				:   int   ):
						# On TPU it's best to pad everything to the same length or training will be very slow.
						__A=  128 if accelerator.distributed_type == DistributedType.TPU else None
						# When using mixed precision we want round multiples of 8/16
						if accelerator.mixed_precision == "fp8":
									__A=  16
						elif accelerator.mixed_precision != "no":
									__A=  8
						else:
									__A=  None

						return tokenizer.pad(
						    _SCREAMING_SNAKE_CASE,padding='longest',max_length=_SCREAMING_SNAKE_CASE,pad_to_multiple_of=_SCREAMING_SNAKE_CASE,return_tensors='pt',)

			# Instantiate dataloaders.
			__A=  DataLoader(
			    tokenized_datasets['train'],shuffle=_SCREAMING_SNAKE_CASE,collate_fn=_SCREAMING_SNAKE_CASE,batch_size=_SCREAMING_SNAKE_CASE   )
			__A=  DataLoader(
			    tokenized_datasets['validation'],shuffle=_SCREAMING_SNAKE_CASE,collate_fn=_SCREAMING_SNAKE_CASE,batch_size=_SCREAMING_SNAKE_CASE   )

			set_seed(_SCREAMING_SNAKE_CASE   )

			# Instantiate the model (we build the model here so that the seed also control new weights initialization)
			__A=  AutoModelForSequenceClassification.from_pretrained(
			    args.model_name_or_path,return_dict=_SCREAMING_SNAKE_CASE,low_cpu_mem_usage=_SCREAMING_SNAKE_CASE   )

			# New Code #
			# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
			__A=  accelerator.prepare(_SCREAMING_SNAKE_CASE   )
			accelerator.print(_SCREAMING_SNAKE_CASE   )

			# Instantiate optimizer
			# New Code #
			# For FSDP feature, at present it doesn't support multiple parameter groups,
			# so we need to create a single parameter group for the whole model
			__A=  torch.optim.AdamW(params=model.parameters(),lr=_SCREAMING_SNAKE_CASE,weight_decay=2e-4   )

			# Instantiate scheduler
			__A=  get_linear_schedule_with_warmup(
			    optimizer=_SCREAMING_SNAKE_CASE,num_warmup_steps=10,num_training_steps=(len(_SCREAMING_SNAKE_CASE   ) * num_epochs) // gradient_accumulation_steps,)

			# New Code #
			# For FSDP feature, prepare everything except the model as we have already prepared the model
			# before creating the optimizer
			# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
			# prepare method.
			__A,  __A,  __A,  __A=  accelerator.prepare(
			    _SCREAMING_SNAKE_CASE,_SCREAMING_SNAKE_CASE,_SCREAMING_SNAKE_CASE,_SCREAMING_SNAKE_CASE   )

			__A=  0

			# Potentially load in the weights and states from a previous save
			if args.resume_from_checkpoint:
						if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
									accelerator.print(f"""Resumed from checkpoint: {args.resume_from_checkpoint}"""   )
									accelerator.load_state(args.resume_from_checkpoint   )
									__A=  os.path.basename(args.resume_from_checkpoint   )
						else:
									# Get the most recent checkpoint
									__A=  [f.name for f in os.scandir(os.getcwd()   ) if f.is_dir()]
									dirs.sort(key=os.path.getctime   )
									__A=  dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
						# Extract `epoch_{i}` or `step_{i}`
						__A=  os.path.splitext(_SCREAMING_SNAKE_CASE   )[0]

						if "epoch" in training_difference:
									num_epochs -= int(training_difference.replace('epoch_',''   )   )
									__A=  None
						else:
									__A=  int(training_difference.replace('step_',''   )   )
									num_epochs -= resume_step // len(_SCREAMING_SNAKE_CASE   )
									# If resuming by step, we also need to know exactly how far into the DataLoader we went
									__A=  (num_epochs * len(_SCREAMING_SNAKE_CASE   )) - resume_step

    # Now we train the model
			for epoch in range(_SCREAMING_SNAKE_CASE   ):
						# New Code #
						# context manager to track the peak memory usage during the training epoch
						with TorchTracemalloc() as tracemalloc:
									model.train()
									if args.with_tracking:
												__A=  0
									for step, batch in enumerate(_SCREAMING_SNAKE_CASE   ):
												# We need to skip steps until we reach the resumed step
												if args.resume_from_checkpoint and epoch == 0:
															if resume_step is not None and step < resume_step:
																		pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
												batch.to(accelerator.device   )
												__A=  model(**_SCREAMING_SNAKE_CASE   )
												__A=  outputs.loss
												__A=  loss / gradient_accumulation_steps
												# We keep track of the loss at each epoch
												if args.with_tracking:
															total_loss += loss.detach().float()
												accelerator.backward(_SCREAMING_SNAKE_CASE   )
												if step % gradient_accumulation_steps == 0:
															optimizer.step()
															lr_scheduler.step()
															optimizer.zero_grad()
															# accelerator.print(lr_scheduler.get_lr())

												overall_step += 1

												if isinstance(_SCREAMING_SNAKE_CASE,_SCREAMING_SNAKE_CASE   ):
															__A=  f"""step_{overall_step}"""
															if overall_step % checkpointing_steps == 0:
																		if args.output_dir is not None:
																					__A=  os.path.join(args.output_dir,_SCREAMING_SNAKE_CASE   )
																		accelerator.save_state(_SCREAMING_SNAKE_CASE   )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
						accelerator.print('Memory before entering the train : {}'.format(bamb(tracemalloc.begin   )   )   )
						accelerator.print('Memory consumed at the end of the train (end-begin): {}'.format(tracemalloc.used   )   )
						accelerator.print('Peak Memory consumed during the train (max-begin): {}'.format(tracemalloc.peaked   )   )
						accelerator.print(
						    'Total Peak Memory consumed during the train (max): {}'.format(
						        tracemalloc.peaked + bamb(tracemalloc.begin   )   )   )
						# Logging the peak memory usage of the GPU to the tracker
						if args.with_tracking:
									accelerator.log(
									    {
									        'train_total_peak_memory': tracemalloc.peaked + bamb(tracemalloc.begin   ),
									    },step=_SCREAMING_SNAKE_CASE,)

						# New Code #
						# context manager to track the peak memory usage during the evaluation
						with TorchTracemalloc() as tracemalloc:
									model.eval()
									for step, batch in enumerate(_SCREAMING_SNAKE_CASE   ):
												# We could avoid this line since we set the accelerator with `device_placement=True`.
												batch.to(accelerator.device   )
												with torch.no_grad():
															__A=  model(**_SCREAMING_SNAKE_CASE   )
												__A=  outputs.logits.argmax(dim=-1   )
												__A,  __A=  accelerator.gather_for_metrics((predictions, batch['labels'])   )
												metric.add_batch(
												    predictions=_SCREAMING_SNAKE_CASE,references=_SCREAMING_SNAKE_CASE,)

									__A=  metric.compute()
									# Use accelerator.print to print only on the main process.
									accelerator.print(f"""epoch {epoch}:""",_SCREAMING_SNAKE_CASE   )
									if args.with_tracking:
												accelerator.log(
												    {
												        'accuracy': eval_metric['accuracy'],
												        'f1': eval_metric['f1'],
												        'train_loss': total_loss.item() / len(_SCREAMING_SNAKE_CASE   ),
												    },step=_SCREAMING_SNAKE_CASE,)

									if checkpointing_steps == "epoch":
												__A=  f"""epoch_{epoch}"""
												if args.output_dir is not None:
															__A=  os.path.join(args.output_dir,_SCREAMING_SNAKE_CASE   )
												accelerator.save_state(_SCREAMING_SNAKE_CASE   )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
						accelerator.print('Memory before entering the eval : {}'.format(bamb(tracemalloc.begin   )   )   )
						accelerator.print('Memory consumed at the end of the eval (end-begin): {}'.format(tracemalloc.used   )   )
						accelerator.print('Peak Memory consumed during the eval (max-begin): {}'.format(tracemalloc.peaked   )   )
						accelerator.print(
						    'Total Peak Memory consumed during the eval (max): {}'.format(tracemalloc.peaked + bamb(tracemalloc.begin   )   )   )
						# Logging the peak memory usage of the GPU to the tracker
						if args.with_tracking:
									accelerator.log(
									    {
									        'eval_total_peak_memory': tracemalloc.peaked + bamb(tracemalloc.begin   ),
									    },step=_SCREAMING_SNAKE_CASE,)

			if args.with_tracking:
						accelerator.end_training()






def 	UpperCAmelCase__(							):






			"""simple docstring"""




			__A=  argparse.ArgumentParser(description='Simple example of training script.'   )
			parser.add_argument(
			    '--mixed_precision',type=_SCREAMING_SNAKE_CASE,default=_SCREAMING_SNAKE_CASE,choices=['no', 'fp16', 'bf16', 'fp8'],help='Whether to use mixed precision. Choose'
			    'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
			    'and an Nvidia Ampere GPU.',)
			parser.add_argument('--cpu',action='store_true',help='If passed, will train on the CPU.'   )
			parser.add_argument(
			    '--checkpointing_steps',type=_SCREAMING_SNAKE_CASE,default=_SCREAMING_SNAKE_CASE,help='Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.',)
			parser.add_argument(
			    '--resume_from_checkpoint',type=_SCREAMING_SNAKE_CASE,default=_SCREAMING_SNAKE_CASE,help='If the training should continue from a checkpoint folder.',)
			parser.add_argument(
			    '--with_tracking',action='store_true',help='Whether to load in all available experiment trackers from the environment and use them for logging.',)
			parser.add_argument(
			    '--output_dir',type=_SCREAMING_SNAKE_CASE,default='.',help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.',)
			parser.add_argument(
			    '--logging_dir',type=_SCREAMING_SNAKE_CASE,default='logs',help='Location on where to store experiment tracking logs`',)
			parser.add_argument(
			    '--model_name_or_path',type=_SCREAMING_SNAKE_CASE,help='Path to pretrained model or model identifier from huggingface.co/models.',required=_SCREAMING_SNAKE_CASE,)
			__A=  parser.parse_args()
			__A=  {'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
			training_function(_SCREAMING_SNAKE_CASE,_SCREAMING_SNAKE_CASE   )


if __name__ == "__main__":
					main()



