




"""simple docstring"""





import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to combine both the gradient accumulation
# and automatic batch size finder utilities of Accelerate to perfrom
# automatic gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

_lowercase						   =     32

def 		lowerCAmelCase				(			lowerCamelCase   ,      lowerCamelCase = 16		):
						'''simple docstring'''


						snake_case__							:       Any				      =       AutoTokenizer.from_pretrained('''bert-base-cased'''		)
						snake_case__							:       Dict				      =       load_dataset('''glue'''   ,      '''mrpc'''		)

						def tokenize_function(lowerCamelCase		):
												# max_length=None => use the model max length (it's actually the default)
												snake_case__							:       Tuple				      =       tokenizer(examples['''sentence1''']   ,      examples['''sentence2''']   ,      truncation=lowerCamelCase   ,      max_length=lowerCamelCase		)
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												snake_case__							:       str				      =       datasets.map(
												    lowerCamelCase   ,      batched=lowerCamelCase   ,      remove_columns=['''idx''', '''sentence1''', '''sentence2''']   ,      )

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						snake_case__							:       List[Any]				      =       tokenized_datasets.rename_column('''label'''   ,      '''labels'''		)

						def collate_fn(lowerCamelCase		):
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		snake_case__							:       str				      =       16
												elif accelerator.mixed_precision != "no":
																		snake_case__							:       Tuple				      =       8
												else:
																		snake_case__							:       List[Any]				      =       None

												return tokenizer.pad(
												    lowerCamelCase   ,      padding='''longest'''   ,      pad_to_multiple_of=lowerCamelCase   ,      return_tensors='''pt'''   ,      )

						# Instantiate dataloaders.
						snake_case__							:       List[str]				      =       DataLoader(
						    tokenized_datasets['''train''']   ,      shuffle=lowerCamelCase   ,      collate_fn=lowerCamelCase   ,      batch_size=lowerCamelCase		)
						snake_case__							:       Tuple				      =       DataLoader(
						    tokenized_datasets['''validation''']   ,      shuffle=lowerCamelCase   ,      collate_fn=lowerCamelCase   ,      batch_size=lowerCamelCase		)

						return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
							from accelerate.test_utils.training import mocked_dataloaders

							_lowercase						   =     mocked_dataloaders  # noqa: F811

def 		lowerCAmelCase				(			lowerCamelCase   ,      lowerCamelCase		):
						'''simple docstring'''


						if os.environ.get('''TESTING_MOCKED_DATALOADERS'''   ,      lowerCamelCase		) == "1":
												snake_case__							:       Any				      =       2
						# Initialize accelerator
						snake_case__							:       Any				      =       Accelerator(cpu=args.cpu   ,      mixed_precision=args.mixed_precision		)
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						snake_case__							:       Any				      =       config['''lr''']
						snake_case__							:       Optional[Any]				      =       int(config['''num_epochs''']		)
						snake_case__							:       str				      =       int(config['''seed''']		)
						snake_case__							:       List[Any]				      =       int(config['''batch_size''']		)

						snake_case__							:       Any				      =       evaluate.load('''glue'''   ,      '''mrpc'''		)

						# New Code #
						# We use the `find_executable_batch_size` decorator, passing in the desired observed batch size
						# to train on. If a CUDA OOM error occurs, it will retry this loop cutting the batch size in
						# half each time. From this, we can calculate the number of gradient accumulation steps needed
						# and modify the Accelerator object as a result
						@find_executable_batch_size(starting_batch_size=int(lowerCamelCase		)		)
						def inner_training_loop(lowerCamelCase		):
												# Since we need to modify the outside accelerator object, we need to bring it
												# to the local scope
												nonlocal accelerator

												# We can calculate the number of gradient accumulation steps based on the current
												# batch size vs the starting batch size
												snake_case__							:       Optional[Any]				      =       observed_batch_size // batch_size

												# And then set it in the Accelerator directly:
												snake_case__							:       Tuple				      =       num_gradient_accumulation_steps

												# Next we need to free all of the stored model references in the Accelerator each time
												accelerator.free_memory()

												# And set the seed so our results are reproducable each reset
												set_seed(lowerCamelCase		)

												# Instantiate the model (we build the model here so that the seed also control new weights initialization)
												snake_case__							:       int				      =       AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''   ,      return_dict=lowerCamelCase		)

												# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
												# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
												# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
												snake_case__							:       str				      =       model.to(accelerator.device		)

												# Instantiate optimizer
												snake_case__							:       Union[str, Any]				      =       AdamW(params=model.parameters()   ,      lr=lowerCamelCase		)
												snake_case__      ,						snake_case__							:       Union[str, Any]				      =       get_dataloaders(lowerCamelCase   ,      lowerCamelCase		)

												# Instantiate scheduler
												snake_case__							:       str				      =       get_linear_schedule_with_warmup(
												    optimizer=lowerCamelCase   ,      num_warmup_steps=100   ,      num_training_steps=(len(lowerCamelCase		) * num_epochs)   ,      )

												# Prepare everything
												# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
												# prepare method.
												snake_case__      ,						snake_case__      ,						snake_case__      ,						snake_case__      ,						snake_case__							:       int				      =       accelerator.prepare(
												    lowerCamelCase   ,      lowerCamelCase   ,      lowerCamelCase   ,      lowerCamelCase   ,      lowerCamelCase		)

												# Now we train the model
												for epoch in range(lowerCamelCase		):
																		model.train()
																		for step, batch in enumerate(lowerCamelCase		):
																								# And perform gradient accumulation
																								with accelerator.accumulate(lowerCamelCase		):
																														# We could avoid this line since we set the accelerator with `device_placement=True`.
																														batch.to(accelerator.device		)
																														snake_case__							:       Tuple				      =       model(**lowerCamelCase		)
																														snake_case__							:       Tuple				      =       outputs.loss
																														accelerator.backward(lowerCamelCase		)
																														optimizer.step()
																														lr_scheduler.step()
																														optimizer.zero_grad()

																		model.eval()
																		for step, batch in enumerate(lowerCamelCase		):
																								# We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device		)
																								with torch.no_grad():
																														snake_case__							:       Optional[int]				      =       model(**lowerCamelCase		)
																								snake_case__							:       List[str]				      =       outputs.logits.argmax(dim=-1		)
																								snake_case__      ,						snake_case__							:       str				      =       accelerator.gather_for_metrics((predictions, batch['''labels'''])		)
																								metric.add_batch(
																								    predictions=lowerCamelCase   ,      references=lowerCamelCase   ,      )

																		snake_case__							:       Optional[Any]				      =       metric.compute()
																		# Use accelerator.print to print only on the main process.
																		accelerator.print(f'''epoch {epoch}:'''   ,      lowerCamelCase		)

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
						inner_training_loop()

def 		lowerCAmelCase				(			):
						'''simple docstring'''


						snake_case__							:       Optional[Any]				      =       argparse.ArgumentParser(description='''Simple example of training script.'''		)
						parser.add_argument(
						    '''--mixed_precision'''   ,      type=lowerCamelCase   ,      default=lowerCamelCase   ,      choices=['''no''', '''fp16''', '''bf16''', '''fp8''']   ,      help='''Whether to use mixed precision. Choose'''
						    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
						    '''and an Nvidia Ampere GPU.'''   ,      )
						parser.add_argument('''--cpu'''   ,      action='''store_true'''   ,      help='''If passed, will train on the CPU.'''		)
						snake_case__							:       Optional[Any]				      =       parser.parse_args()
						# New Code #
						# We modify the starting batch size to be an observed batch size of 256, to guarentee an initial CUDA OOM
						snake_case__							:       List[Any]				      =       {'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 256}
						training_function(lowerCamelCase   ,      lowerCamelCase		)


if __name__ == "__main__":
							main()








"""simple docstring"""





import math
import os

import cva
import numpy as np

_lowercase						   =     2_55.0

def 		lowerCAmelCase				(			lowerCamelCase   ,      lowerCamelCase		):
						'''simple docstring'''


						snake_case__							:       Tuple				      =       np.mean((original - contrast) ** 2		)
						if mse == 0:
												return 100

						return 20 * math.logaa(PIXEL_MAX / math.sqrt(lowerCamelCase		)		)

def 		lowerCAmelCase				(			):
						'''simple docstring'''


						snake_case__							:       Any				      =       os.path.dirname(os.path.realpath(lowerCamelCase		)		)
						# Loading images (original image and compressed image)
						snake_case__							:       Any				      =       cva.imread(os.path.join(lowerCamelCase   ,      '''image_data/original_image.png'''		)		)
						snake_case__							:       Any				      =       cva.imread(os.path.join(lowerCamelCase   ,      '''image_data/compressed_image.png'''		)   ,      1		)

						snake_case__							:       Optional[Any]				      =       cva.imread(os.path.join(lowerCamelCase   ,      '''image_data/PSNR-example-base.png'''		)		)
						snake_case__							:       Tuple				      =       cva.imread(
						    os.path.join(lowerCamelCase   ,      '''image_data/PSNR-example-comp-10.jpg'''		)   ,      1		)

						# Value expected: 29.73dB
						print('''-- First Test --'''		)
						print(f'''PSNR value is {peak_signal_to_noise_ratio(lowerCamelCase   ,      lowerCamelCase		)} dB'''		)

						# # Value expected: 31.53dB (Wikipedia Example)
						print('''\n-- Second Test --'''		)
						print(f'''PSNR value is {peak_signal_to_noise_ratio(lowerCamelCase   ,      lowerCamelCase		)} dB'''		)


if __name__ == "__main__":
							main()


