
'''simple docstring'''







from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_lowercase				:						List[str]												= logging.get_logger(__name__)

_lowercase				:						Any												= {
    "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json",
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}






class 				lowerCAmelCase__     (						lowerCamelCase_  ):
					lowerCAmelCase_       =  '''m2m_100'''
					lowerCAmelCase_       =  ['''past_key_values''']
					lowerCAmelCase_       =  {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE=12_81_12       ,							__SCREAMING_SNAKE_CASE=10_24       ,							__SCREAMING_SNAKE_CASE=12       ,							__SCREAMING_SNAKE_CASE=40_96       ,							__SCREAMING_SNAKE_CASE=16       ,							__SCREAMING_SNAKE_CASE=12       ,							__SCREAMING_SNAKE_CASE=40_96       ,							__SCREAMING_SNAKE_CASE=16       ,							__SCREAMING_SNAKE_CASE=0.05       ,							__SCREAMING_SNAKE_CASE=0.05       ,							__SCREAMING_SNAKE_CASE=True       ,							__SCREAMING_SNAKE_CASE=True       ,							__SCREAMING_SNAKE_CASE="relu"       ,							__SCREAMING_SNAKE_CASE=10_24       ,							__SCREAMING_SNAKE_CASE=0.1       ,							__SCREAMING_SNAKE_CASE=0.1       ,							__SCREAMING_SNAKE_CASE=0.0       ,							__SCREAMING_SNAKE_CASE=0.02       ,							__SCREAMING_SNAKE_CASE=2       ,							__SCREAMING_SNAKE_CASE=True       ,							__SCREAMING_SNAKE_CASE=1       ,							__SCREAMING_SNAKE_CASE=0       ,							__SCREAMING_SNAKE_CASE=2       ,							**__SCREAMING_SNAKE_CASE       ,							):


											"""simple docstring"""





											lowercase_			:  Optional[int]									= vocab_size
											lowercase_			:  List[str]									= max_position_embeddings
											lowercase_			:  Optional[Any]									= d_model
											lowercase_			:  Optional[int]									= encoder_ffn_dim
											lowercase_			:  Any									= encoder_layers
											lowercase_			:  List[Any]									= encoder_attention_heads
											lowercase_			:  int									= decoder_ffn_dim
											lowercase_			:  int									= decoder_layers
											lowercase_			:  List[str]									= decoder_attention_heads
											lowercase_			:  Optional[int]									= dropout
											lowercase_			:  Tuple									= attention_dropout
											lowercase_			:  Tuple									= activation_dropout
											lowercase_			:  Optional[Any]									= activation_function
											lowercase_			:  List[str]									= init_std
											lowercase_			:  Dict									= encoder_layerdrop
											lowercase_			:  Optional[int]									= decoder_layerdrop
											lowercase_			:  List[Any]									= use_cache
											lowercase_			:  Optional[Any]									= encoder_layers
											lowercase_			:  Tuple									= scale_embedding  # scale factor will be sqrt(d_model) if True

											super().__init__(
											    pad_token_id=__SCREAMING_SNAKE_CASE       ,							bos_token_id=__SCREAMING_SNAKE_CASE       ,							eos_token_id=__SCREAMING_SNAKE_CASE       ,							is_encoder_decoder=__SCREAMING_SNAKE_CASE       ,							decoder_start_token_id=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE       ,							)







class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					@property
					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Tuple									= OrderedDict(
											    [
											        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
											        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
											    ]	)

											if self.use_past:
																	lowercase_			:  Optional[Any]									= {0: '''batch'''}
																	lowercase_			:  List[Any]									= {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
											else:
																	lowercase_			:  Any									= {0: '''batch''', 1: '''decoder_sequence'''}
																	lowercase_			:  Tuple									= {0: '''batch''', 1: '''decoder_sequence'''}

											if self.use_past:
																	self.fill_with_past_key_values_(__SCREAMING_SNAKE_CASE       ,							direction='''inputs'''	)
											return common_inputs






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = -1       ,							__SCREAMING_SNAKE_CASE = -1       ,							__SCREAMING_SNAKE_CASE = False       ,							__SCREAMING_SNAKE_CASE = None       ,							):


											"""simple docstring"""





											lowercase_			:  List[Any]									= compute_effective_axis_dimension(
											    __SCREAMING_SNAKE_CASE       ,							fixed_dimension=OnnxConfig.default_fixed_batch       ,							num_token_to_add=0	)

											# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
											lowercase_			:  int									= tokenizer.num_special_tokens_to_add(__SCREAMING_SNAKE_CASE	)
											lowercase_			:  Tuple									= compute_effective_axis_dimension(
											    __SCREAMING_SNAKE_CASE       ,							fixed_dimension=OnnxConfig.default_fixed_sequence       ,							num_token_to_add=__SCREAMING_SNAKE_CASE	)

											# Generate dummy inputs according to compute batch and sequence
											lowercase_			:  int									= [''' '''.join([tokenizer.unk_token]	) * seq_length] * batch_size
											lowercase_			:  int									= dict(tokenizer(__SCREAMING_SNAKE_CASE       ,							return_tensors=__SCREAMING_SNAKE_CASE	)	)
											return common_inputs






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = -1       ,							__SCREAMING_SNAKE_CASE = -1       ,							__SCREAMING_SNAKE_CASE = False       ,							__SCREAMING_SNAKE_CASE = None       ,							):


											"""simple docstring"""





											lowercase_			:  str									= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
											    __SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)

											# Generate decoder inputs
											lowercase_			:  Dict									= seq_length if not self.use_past else 1
											lowercase_			:  Optional[Any]									= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
											    __SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)
											lowercase_			:  str									= {F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
											lowercase_			:  Optional[int]									= dict(**__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)

											if self.use_past:
																	if not is_torch_available():
																							raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''	)
																	else:
																							import torch
																	lowercase_		,				lowercase_			:  List[Any]									= common_inputs['''input_ids'''].shape
																	lowercase_			:  Tuple									= common_inputs['''decoder_input_ids'''].shape[1]
																	lowercase_		,				lowercase_			:  List[str]									= self.num_attention_heads
																	lowercase_			:  int									= (
																	    batch,
																	    num_encoder_attention_heads,
																	    encoder_seq_length,
																	    self._config.hidden_size // num_encoder_attention_heads,
																	)
																	lowercase_			:  Dict									= decoder_seq_length + 3
																	lowercase_			:  int									= (
																	    batch,
																	    num_decoder_attention_heads,
																	    decoder_past_length,
																	    self._config.hidden_size // num_decoder_attention_heads,
																	)

																	lowercase_			:  List[Any]									= torch.cat(
																	    [common_inputs['''decoder_attention_mask'''], torch.ones(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)]       ,							dim=1	)

																	lowercase_			:  List[Any]									= []
																	# If the number of encoder and decoder layers are present in the model configuration, both are considered
																	lowercase_		,				lowercase_			:  str									= self.num_layers
																	lowercase_			:  List[str]									= min(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)
																	lowercase_			:  str									= max(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	) - min_num_layers
																	lowercase_			:  List[str]									= '''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

																	for _ in range(__SCREAMING_SNAKE_CASE	):
																							common_inputs["past_key_values"].append(
																							    (
																							        torch.zeros(__SCREAMING_SNAKE_CASE	),
																							        torch.zeros(__SCREAMING_SNAKE_CASE	),
																							        torch.zeros(__SCREAMING_SNAKE_CASE	),
																							        torch.zeros(__SCREAMING_SNAKE_CASE	),
																							    )	)
																	# TODO: test this.
																	lowercase_			:  int									= encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
																	for _ in range(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):
																							common_inputs["past_key_values"].append((torch.zeros(__SCREAMING_SNAKE_CASE	), torch.zeros(__SCREAMING_SNAKE_CASE	))	)
											return common_inputs




					lowerCAmelCase_       =  _generate_dummy_inputs_for_default_and_seqaseq_lm





'''simple docstring'''







import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to combine both the gradient accumulation
# and automatic batch size finder utilities of Accelerate to perfrom
# automatic gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

_lowercase				:						Tuple												= 3_2
def       snake_case_	(       __SCREAMING_SNAKE_CASE      :				Accelerator   ,   __SCREAMING_SNAKE_CASE      :				int = 16							):



						"""simple docstring"""





						lowercase_			:  int									= AutoTokenizer.from_pretrained('''bert-base-cased'''							)
						lowercase_			:  int									= load_dataset('''glue'''   ,   '''mrpc'''							)

						def tokenize_function(__SCREAMING_SNAKE_CASE      :				Optional[int]							):
												# max_length=None => use the model max length (it's actually the default)
												lowercase_			:  Union[str, Any]									= tokenizer(examples['''sentence1''']   ,   examples['''sentence2''']   ,   truncation=__SCREAMING_SNAKE_CASE   ,   max_length=__SCREAMING_SNAKE_CASE							)
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												lowercase_			:  Optional[Any]									= datasets.map(
												    __SCREAMING_SNAKE_CASE   ,   batched=__SCREAMING_SNAKE_CASE   ,   remove_columns=['''idx''', '''sentence1''', '''sentence2''']   ,   )

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						lowercase_			:  Any									= tokenized_datasets.rename_column('''label'''   ,   '''labels'''							)

						def collate_fn(__SCREAMING_SNAKE_CASE      :				List[Any]							):
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		lowercase_			:  List[str]									= 16
												elif accelerator.mixed_precision != "no":
																		lowercase_			:  List[Any]									= 8
												else:
																		lowercase_			:  List[Any]									= None

												return tokenizer.pad(
												    __SCREAMING_SNAKE_CASE   ,   padding='''longest'''   ,   pad_to_multiple_of=__SCREAMING_SNAKE_CASE   ,   return_tensors='''pt'''   ,   )

						# Instantiate dataloaders.
						lowercase_			:  List[str]									= DataLoader(
						    tokenized_datasets['''train''']   ,   shuffle=__SCREAMING_SNAKE_CASE   ,   collate_fn=__SCREAMING_SNAKE_CASE   ,   batch_size=__SCREAMING_SNAKE_CASE							)
						lowercase_			:  int									= DataLoader(
						    tokenized_datasets['''validation''']   ,   shuffle=__SCREAMING_SNAKE_CASE   ,   collate_fn=__SCREAMING_SNAKE_CASE   ,   batch_size=__SCREAMING_SNAKE_CASE							)

						return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
							from accelerate.test_utils.training import mocked_dataloaders

							_lowercase				:						Tuple												= mocked_dataloaders  # noqa: F811
def       snake_case_	(       __SCREAMING_SNAKE_CASE      :				str   ,   __SCREAMING_SNAKE_CASE      :				List[Any]							):



						"""simple docstring"""





						if os.environ.get('''TESTING_MOCKED_DATALOADERS'''   ,   __SCREAMING_SNAKE_CASE							) == "1":
												lowercase_			:  Dict									= 2
						# Initialize accelerator
						lowercase_			:  Any									= Accelerator(cpu=args.cpu   ,   mixed_precision=args.mixed_precision							)
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						lowercase_			:  Optional[Any]									= config['''lr''']
						lowercase_			:  Optional[Any]									= int(config['''num_epochs''']							)
						lowercase_			:  str									= int(config['''seed''']							)
						lowercase_			:  Optional[Any]									= int(config['''batch_size''']							)

						lowercase_			:  int									= evaluate.load('''glue'''   ,   '''mrpc'''							)

						# New Code #
						# We use the `find_executable_batch_size` decorator, passing in the desired observed batch size
						# to train on. If a CUDA OOM error occurs, it will retry this loop cutting the batch size in
						# half each time. From this, we can calculate the number of gradient accumulation steps needed
						# and modify the Accelerator object as a result
						@find_executable_batch_size(starting_batch_size=int(__SCREAMING_SNAKE_CASE							)							)
						def inner_training_loop(__SCREAMING_SNAKE_CASE      :				Any							):
												# Since we need to modify the outside accelerator object, we need to bring it
												# to the local scope
												nonlocal accelerator

												# We can calculate the number of gradient accumulation steps based on the current
												# batch size vs the starting batch size
												lowercase_			:  Union[str, Any]									= observed_batch_size // batch_size

												# And then set it in the Accelerator directly:
												lowercase_			:  Tuple									= num_gradient_accumulation_steps

												# Next we need to free all of the stored model references in the Accelerator each time
												accelerator.free_memory()

												# And set the seed so our results are reproducable each reset
												set_seed(__SCREAMING_SNAKE_CASE							)

												# Instantiate the model (we build the model here so that the seed also control new weights initialization)
												lowercase_			:  Dict									= AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''   ,   return_dict=__SCREAMING_SNAKE_CASE							)

												# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
												# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
												# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
												lowercase_			:  Optional[int]									= model.to(accelerator.device							)

												# Instantiate optimizer
												lowercase_			:  int									= AdamW(params=model.parameters()   ,   lr=__SCREAMING_SNAKE_CASE							)
												lowercase_		,				lowercase_			:  List[str]									= get_dataloaders(__SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE							)

												# Instantiate scheduler
												lowercase_			:  str									= get_linear_schedule_with_warmup(
												    optimizer=__SCREAMING_SNAKE_CASE   ,   num_warmup_steps=100   ,   num_training_steps=(len(__SCREAMING_SNAKE_CASE							) * num_epochs)   ,   )

												# Prepare everything
												# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
												# prepare method.
												lowercase_		,				lowercase_		,				lowercase_		,				lowercase_		,				lowercase_			:  Any									= accelerator.prepare(
												    __SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE							)

												# Now we train the model
												for epoch in range(__SCREAMING_SNAKE_CASE							):
																		model.train()
																		for step, batch in enumerate(__SCREAMING_SNAKE_CASE							):
																								# And perform gradient accumulation
																								with accelerator.accumulate(__SCREAMING_SNAKE_CASE							):
																														# We could avoid this line since we set the accelerator with `device_placement=True`.
																														batch.to(accelerator.device							)
																														lowercase_			:  List[str]									= model(**__SCREAMING_SNAKE_CASE							)
																														lowercase_			:  Any									= outputs.loss
																														accelerator.backward(__SCREAMING_SNAKE_CASE							)
																														optimizer.step()
																														lr_scheduler.step()
																														optimizer.zero_grad()

																		model.eval()
																		for step, batch in enumerate(__SCREAMING_SNAKE_CASE							):
																								# We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device							)
																								with torch.no_grad():
																														lowercase_			:  Tuple									= model(**__SCREAMING_SNAKE_CASE							)
																								lowercase_			:  List[str]									= outputs.logits.argmax(dim=-1							)
																								lowercase_		,				lowercase_			:  int									= accelerator.gather_for_metrics((predictions, batch['''labels'''])							)
																								metric.add_batch(
																								    predictions=__SCREAMING_SNAKE_CASE   ,   references=__SCREAMING_SNAKE_CASE   ,   )

																		lowercase_			:  Union[str, Any]									= metric.compute()
																		# Use accelerator.print to print only on the main process.
																		accelerator.print(F'''epoch {epoch}:'''   ,   __SCREAMING_SNAKE_CASE							)

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
						inner_training_loop()
def       snake_case_	(       ):



						"""simple docstring"""





						lowercase_			:  int									= argparse.ArgumentParser(description='''Simple example of training script.'''							)
						parser.add_argument(
						    '''--mixed_precision'''   ,   type=__SCREAMING_SNAKE_CASE   ,   default=__SCREAMING_SNAKE_CASE   ,   choices=['''no''', '''fp16''', '''bf16''', '''fp8''']   ,   help='''Whether to use mixed precision. Choose'''
						    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
						    '''and an Nvidia Ampere GPU.'''   ,   )
						parser.add_argument('''--cpu'''   ,   action='''store_true'''   ,   help='''If passed, will train on the CPU.'''							)
						lowercase_			:  int									= parser.parse_args()
						# New Code #
						# We modify the starting batch size to be an observed batch size of 256, to guarentee an initial CUDA OOM
						lowercase_			:  Union[str, Any]									= {'''lr''': 2E-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 256}
						training_function(__SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE							)


if __name__ == "__main__":
							main()



