





import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing the checkpointing capability,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

__lowercase													=			16
__lowercase													=			32


def 					__SCREAMING_SNAKE_CASE  (							__lowercase:		Optional[int]     ,							__lowercase:		Optional[Any] = 16):



  '''simple docstring'''






  __lowerCAmelCase			=				AutoTokenizer.from_pretrained('''bert-base-cased''')
  __lowerCAmelCase			=				load_dataset('''glue'''     ,							'''mrpc''')

  def tokenize_function(__lowercase:		int):
    # max_length=None => use the model max length (it's actually the default)
    __lowerCAmelCase			=				tokenizer(examples['''sentence1''']     ,							examples['''sentence2''']     ,							truncation=__lowercase     ,							max_length=__lowercase)
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  # starting with the main process first:
  with accelerator.main_process_first():
    __lowerCAmelCase			=				datasets.map(
        __lowercase     ,							batched=__lowercase     ,							remove_columns=['''idx''', '''sentence1''', '''sentence2''']     ,							)

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  __lowerCAmelCase			=				tokenized_datasets.rename_column('''label'''     ,							'''labels''')

  def collate_fn(__lowercase:		Tuple):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    __lowerCAmelCase			=				1_28 if accelerator.distributed_type == DistributedType.TPU else None
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
      __lowerCAmelCase			=				16
    elif accelerator.mixed_precision != "no":
      __lowerCAmelCase			=				8
    else:
      __lowerCAmelCase			=				None

    return tokenizer.pad(
        __lowercase     ,							padding='''longest'''     ,							max_length=__lowercase     ,							pad_to_multiple_of=__lowercase     ,							return_tensors='''pt'''     ,							)

  # Instantiate dataloaders.
  __lowerCAmelCase			=				DataLoader(
      tokenized_datasets['''train''']     ,							shuffle=__lowercase     ,							collate_fn=__lowercase     ,							batch_size=__lowercase)
  __lowerCAmelCase			=				DataLoader(
      tokenized_datasets['''validation''']     ,							shuffle=__lowercase     ,							collate_fn=__lowercase     ,							batch_size=__lowercase)

  return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
      from accelerate.test_utils.training import mocked_dataloaders

      __lowercase													=			mocked_dataloaders  # noqa: F811


def 					__SCREAMING_SNAKE_CASE  (							__lowercase:		int     ,							__lowercase:		Optional[int]):



  '''simple docstring'''






  if os.environ.get('''TESTING_MOCKED_DATALOADERS'''     ,							__lowercase) == "1":
    __lowerCAmelCase			=				2
  # Initialize accelerator
  __lowerCAmelCase			=				Accelerator(cpu=args.cpu     ,							mixed_precision=args.mixed_precision)
  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  __lowerCAmelCase			=				config["lr"]
  __lowerCAmelCase			=				int(config['''num_epochs'''])
  __lowerCAmelCase			=				int(config['''seed'''])
  __lowerCAmelCase			=				int(config['''batch_size'''])

  # New Code #
  # Parse out whether we are saving every epoch or after a certain number of batches
  if hasattr(args.checkpointing_steps     ,							'''isdigit'''):
    if args.checkpointing_steps == "epoch":
      __lowerCAmelCase			=				args.checkpointing_steps
    elif args.checkpointing_steps.isdigit():
      __lowerCAmelCase			=				int(args.checkpointing_steps)
    else:
      raise ValueError(
          F'''Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.''')
  else:
    __lowerCAmelCase			=				None

  set_seed(__lowercase)

  __lowerCAmelCase			=				get_dataloaders(__lowercase     ,							__lowercase)
  __lowerCAmelCase			=				evaluate.load('''glue'''     ,							'''mrpc''')

  # If the batch size is too big we use gradient accumulation
  __lowerCAmelCase			=				1
  if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
    __lowerCAmelCase			=				batch_size // MAX_GPU_BATCH_SIZE
    __lowerCAmelCase			=				MAX_GPU_BATCH_SIZE

  # Instantiate the model (we build the model here so that the seed also control new weights initialization)
  __lowerCAmelCase			=				AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''     ,							return_dict=__lowercase)

  # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
  # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
  # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
  __lowerCAmelCase			=				model.to(accelerator.device)

  # Instantiate optimizer
  __lowerCAmelCase			=				AdamW(params=model.parameters()     ,							lr=__lowercase)

  # Instantiate scheduler
  __lowerCAmelCase			=				get_linear_schedule_with_warmup(
      optimizer=__lowercase     ,							num_warmup_steps=1_00     ,							num_training_steps=(len(__lowercase) * num_epochs) // gradient_accumulation_steps     ,							)

  # Prepare everything
  # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  # prepare method.
  __lowerCAmelCase			=				accelerator.prepare(
      __lowercase     ,							__lowercase     ,							__lowercase     ,							__lowercase     ,							__lowercase)

  # New Code #
  # We need to keep track of how many total steps we have iterated over
  __lowerCAmelCase			=				0
  # We also need to keep track of the stating epoch so files are named properly
  __lowerCAmelCase			=				0

  # We need to load the checkpoint back in before training here with `load_state`
  # The total number of epochs is adjusted based on where the state is being loaded from,
  # as we assume continuation of the same training script
  if args.resume_from_checkpoint:
    if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
      accelerator.print(F'''Resumed from checkpoint: {args.resume_from_checkpoint}''')
      accelerator.load_state(args.resume_from_checkpoint)
      __lowerCAmelCase			=				os.path.basename(args.resume_from_checkpoint)
    else:
      # Get the most recent checkpoint
      __lowerCAmelCase			=				[f.name for f in os.scandir(os.getcwd()) if f.is_dir()]
      dirs.sort(key=os.path.getctime)
      __lowerCAmelCase			=				dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
    # Extract `epoch_{i}` or `step_{i}`
    __lowerCAmelCase			=				os.path.splitext(__lowercase)[0]

    if "epoch" in training_difference:
      __lowerCAmelCase			=				int(training_difference.replace('''epoch_'''     ,							'''''')) + 1
      __lowerCAmelCase			=				None
    else:
      __lowerCAmelCase			=				int(training_difference.replace('''step_'''     ,							''''''))
      __lowerCAmelCase			=				resume_step // len(__lowercase)
      resume_step -= starting_epoch * len(__lowercase)

    # Now we train the model
  for epoch in range(__lowercase     ,							__lowercase):
    model.train()
    # New Code #
    if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:
      # We need to skip steps until we reach the resumed step
      __lowerCAmelCase			=				accelerator.skip_first_batches(__lowercase     ,							__lowercase)
      overall_step += resume_step
    else:
      # After the first iteration though, we need to go back to the original dataloader
      __lowerCAmelCase			=				train_dataloader
    for step, batch in enumerate(__lowercase):
      # We could avoid this line since we set the accelerator with `device_placement=True`.
      batch.to(accelerator.device)
      __lowerCAmelCase			=				model(**__lowercase)
      __lowerCAmelCase			=				outputs.loss
      __lowerCAmelCase			=				loss / gradient_accumulation_steps
      accelerator.backward(__lowercase)
      if step % gradient_accumulation_steps == 0:
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
      # New Code #
      overall_step += 1

      # New Code #
      # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
      # These are saved to folders named `step_{overall_step}`
      # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
      # If mixed precision was used, will also save a "scalar.bin" file
      if isinstance(__lowercase     ,							__lowercase):
        __lowerCAmelCase			=				F'''step_{overall_step}'''
        if overall_step % checkpointing_steps == 0:
          if args.output_dir is not None:
            __lowerCAmelCase			=				os.path.join(args.output_dir     ,							__lowercase)
          accelerator.save_state(__lowercase)

    model.eval()
    for step, batch in enumerate(__lowercase):
      # We could avoid this line since we set the accelerator with `device_placement=True` (the default).
      batch.to(accelerator.device)
      with torch.no_grad():
        __lowerCAmelCase			=				model(**__lowercase)
      __lowerCAmelCase			=				outputs.logits.argmax(dim=-1)
      __lowerCAmelCase			=				accelerator.gather_for_metrics((predictions, batch['''labels''']))
      metric.add_batch(
          predictions=__lowercase     ,							references=__lowercase     ,							)

    __lowerCAmelCase			=				metric.compute()
    # Use accelerator.print to print only on the main process.
    accelerator.print(F'''epoch {epoch}:'''     ,							__lowercase)

    # New Code #
    # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
    # These are saved to folders named `epoch_{epoch}`
    # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
    # If mixed precision was used, will also save a "scalar.bin" file
    if checkpointing_steps == "epoch":
      __lowerCAmelCase			=				F'''epoch_{epoch}'''
      if args.output_dir is not None:
        __lowerCAmelCase			=				os.path.join(args.output_dir     ,							__lowercase)
      accelerator.save_state(__lowercase)


def 					__SCREAMING_SNAKE_CASE  (							):



  '''simple docstring'''






  __lowerCAmelCase			=				argparse.ArgumentParser(description='''Simple example of training script.''')
  parser.add_argument(
      '''--mixed_precision'''     ,							type=__lowercase     ,							default=__lowercase     ,							choices=['''no''', '''fp16''', '''bf16''', '''fp8''']     ,							help='''Whether to use mixed precision. Choose'''
      '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
      '''and an Nvidia Ampere GPU.'''     ,							)
  parser.add_argument('''--cpu'''     ,							action='''store_true'''     ,							help='''If passed, will train on the CPU.''')
  parser.add_argument(
      '''--checkpointing_steps'''     ,							type=__lowercase     ,							default=__lowercase     ,							help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''     ,							)
  parser.add_argument(
      '''--output_dir'''     ,							type=__lowercase     ,							default='''.'''     ,							help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''     ,							)
  parser.add_argument(
      '''--resume_from_checkpoint'''     ,							type=__lowercase     ,							default=__lowercase     ,							help='''If the training should continue from a checkpoint folder.'''     ,							)
  __lowerCAmelCase			=				parser.parse_args()
  __lowerCAmelCase			=				{"lr": 2E-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
  training_function(__lowercase     ,							__lowercase)


if __name__ == "__main__":
      main()







import subprocess
from argparse import ArgumentParser
from typing import List, Union

from huggingface_hub.hf_api import HfFolder, create_repo, whoami
from requests.exceptions import HTTPError

from . import BaseTransformersCLICommand

class   __snake_case				(			_a   ):






    '''simple docstring'''





    @staticmethod
    def 	A		(   snake_case						:		ArgumentParser							):

      """simple docstring"""





      __lowerCAmelCase			=				parser.add_parser('''login'''			,						help='''Log in using the same credentials as on huggingface.co'''							)
      login_parser.set_defaults(func=lambda snake_case							: LoginCommand(snake_case							)							)
      __lowerCAmelCase			=				parser.add_parser('''whoami'''			,						help='''Find out which huggingface.co account you are logged in as.'''							)
      whoami_parser.set_defaults(func=lambda snake_case							: WhoamiCommand(snake_case							)							)
      __lowerCAmelCase			=				parser.add_parser('''logout'''			,						help='''Log out'''							)
      logout_parser.set_defaults(func=lambda snake_case							: LogoutCommand(snake_case							)							)

      # new system: git-based repo system
      __lowerCAmelCase			=				parser.add_parser(
          '''repo'''			,						help='''Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos.'''			,						)
      __lowerCAmelCase			=				repo_parser.add_subparsers(
          help='''Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands'''							)
      __lowerCAmelCase			=				repo_subparsers.add_parser(
          '''create'''			,						help='''Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co'''							)
      repo_create_parser.add_argument(
          '''name'''			,						type=snake_case			,						help='''Name for your model\'s repo. Will be namespaced under your username to build the model id.'''			,						)
      repo_create_parser.add_argument('''--organization'''			,						type=snake_case			,						help='''Optional: organization namespace.'''							)
      repo_create_parser.add_argument('''-y'''			,						'''--yes'''			,						action='''store_true'''			,						help='''Optional: answer Yes to the prompt'''							)
      repo_create_parser.set_defaults(func=lambda snake_case							: RepoCreateCommand(snake_case							)							)


class   __snake_case				:






    '''simple docstring'''
    _lowerCAmelCase     =	"""\u001b[1m"""
    _lowerCAmelCase     =	"""\u001b[31m"""
    _lowerCAmelCase     =	"""\u001b[90m"""
    _lowerCAmelCase     =	"""\u001b[0m"""





    @classmethod
    def 	A		(   cls						:		str			,						snake_case						:		Optional[int]							):

      """simple docstring"""





      return f'''{cls._bold}{s}{cls._reset}'''





    @classmethod
    def 	A		(   cls						:		Optional[Any]			,						snake_case						:		str							):

      """simple docstring"""





      return f'''{cls._bold}{cls._red}{s}{cls._reset}'''





    @classmethod
    def 	A		(   cls						:		Dict			,						snake_case						:		List[Any]							):

      """simple docstring"""





      return f'''{cls._gray}{s}{cls._reset}'''


def 					__SCREAMING_SNAKE_CASE  (							__lowercase:		List[List[Union[str, int]]]     ,							__lowercase:		List[str]):



  '''simple docstring'''






  __lowerCAmelCase			=				[max(len(str(__lowercase)) for x in col) for col in zip(*__lowercase     ,							__lowercase)]
  __lowerCAmelCase			=				('''{{:{}}} ''' * len(__lowercase)).format(*__lowercase)
  __lowerCAmelCase			=				[]
  lines.append(row_format.format(*__lowercase))
  lines.append(row_format.format(*['''-''' * w for w in col_widths]))
  for row in rows:
    lines.append(row_format.format(*__lowercase))
  return "\n".join(__lowercase)

class   __snake_case				:






    '''simple docstring'''





    def __init__(   self						:		List[str]			,						snake_case						:		List[str]							):

      """simple docstring"""





      __lowerCAmelCase			=				args


class   __snake_case				(			_a   ):






    '''simple docstring'''





    def 	A		(   self						:		List[Any]							):

      """simple docstring"""





      print(
          ANSI.red(
              '''ERROR! `huggingface-cli login` uses an outdated login mechanism '''
              '''that is not compatible with the Hugging Face Hub backend anymore. '''
              '''Please use `huggingface-cli login instead.'''							)							)


class   __snake_case				(			_a   ):






    '''simple docstring'''





    def 	A		(   self						:		int							):

      """simple docstring"""





      print(
          ANSI.red(
              '''WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use '''
              '''`huggingface-cli whoami` instead.'''							)							)
      __lowerCAmelCase			=				HfFolder.get_token()
      if token is None:
        print('''Not logged in'''							)
        exit()
      try:
        __lowerCAmelCase	,						__lowerCAmelCase			=				whoami(snake_case							)
        print(snake_case							)
        if orgs:
          print(ANSI.bold('''orgs: '''							)			,						''','''.join(snake_case							)							)
      except HTTPError as e:
        print(snake_case							)
        print(ANSI.red(e.response.text							)							)
        exit(1							)


class   __snake_case				(			_a   ):






    '''simple docstring'''





    def 	A		(   self						:		Any							):

      """simple docstring"""





      print(
          ANSI.red(
              '''ERROR! `transformers-cli logout` uses an outdated logout mechanism '''
              '''that is not compatible with the Hugging Face Hub backend anymore. '''
              '''Please use `huggingface-cli logout instead.'''							)							)


class   __snake_case				(			_a   ):






    '''simple docstring'''





    def 	A		(   self						:		Optional[int]							):

      """simple docstring"""





      print(
          ANSI.red(
              '''WARNING! Managing repositories through transformers-cli is deprecated. '''
              '''Please use `huggingface-cli` instead.'''							)							)
      __lowerCAmelCase			=				HfFolder.get_token()
      if token is None:
        print('''Not logged in'''							)
        exit(1							)
      try:
        __lowerCAmelCase			=				subprocess.check_output(['''git''', '''--version''']							).decode('''utf-8'''							)
        print(ANSI.gray(stdout.strip()							)							)
      except FileNotFoundError:
        print('''Looks like you do not have git installed, please install.'''							)

      try:
        __lowerCAmelCase			=				subprocess.check_output(['''git-lfs''', '''--version''']							).decode('''utf-8'''							)
        print(ANSI.gray(stdout.strip()							)							)
      except FileNotFoundError:
        print(
            ANSI.red(
                '''Looks like you do not have git-lfs installed, please install.'''
                ''' You can install from https://git-lfs.github.com/.'''
                ''' Then run `git lfs install` (you only have to do this once).'''							)							)
      print(''''''							)

      __lowerCAmelCase	,						__lowerCAmelCase			=				whoami(snake_case							)
      __lowerCAmelCase			=				self.args.organization if self.args.organization is not None else user
      __lowerCAmelCase			=				f'''{namespace}/{self.args.name}'''
      print(f'''You are about to create {ANSI.bold(snake_case							)}'''							)

      if not self.args.yes:
        __lowerCAmelCase			=				input('''Proceed? [Y/n] '''							).lower()
        if not (choice == "" or choice == "y" or choice == "yes"):
          print('''Abort'''							)
          exit()
      try:
        __lowerCAmelCase			=				create_repo(snake_case			,						name=self.args.name			,						organization=self.args.organization							)
      except HTTPError as e:
        print(snake_case							)
        print(ANSI.red(e.response.text							)							)
        exit(1							)
      print('''\nYour repo now lives at:'''							)
      print(f'''  {ANSI.bold(snake_case							)}'''							)
      print('''\nYou can clone it locally with the command below, and commit/push as usual.'''							)
      print(f'''\n  git clone {url}'''							)
      print(''''''							)
