




'''simple docstring'''





import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


a		:	Any       =      logging.getLogger(__name__)

require_version('''pytorch_lightning>=1.0.4''')

a		:	Optional[Any]       =      {
    '''base''': AutoModel,
    '''sequence-classification''': AutoModelForSequenceClassification,
    '''question-answering''': AutoModelForQuestionAnswering,
    '''pretraining''': AutoModelForPreTraining,
    '''token-classification''': AutoModelForTokenClassification,
    '''language-modeling''': AutoModelWithLMHead,
    '''summarization''': AutoModelForSeqaSeqLM,
    '''translation''': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
a		:	Optional[Any]       =      {
    '''linear''': get_linear_schedule_with_warmup,
    '''cosine''': get_cosine_schedule_with_warmup,
    '''cosine_w_restarts''': get_cosine_with_hard_restarts_schedule_with_warmup,
    '''polynomial''': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
a		:	Dict       =      sorted(arg_to_scheduler.keys())
a		:	Tuple       =      '''{''' + ''', '''.join(arg_to_scheduler_choices) + '''}'''




class      UpperCamelCase       (	pl.LightningModule       ):



  def __init__(			self		,     __lowerCAmelCase		,     __lowerCAmelCase=None		,     __lowerCAmelCase="base"		,     __lowerCAmelCase=None		,     __lowerCAmelCase=None		,     __lowerCAmelCase=None		,     **__lowerCAmelCase		,     ):







      '''simple docstring'''


      super().__init__()
      # TODO: move to self.save_hyperparameters()
      # self.save_hyperparameters()
      # can also expand arguments into trainer signature for easier reading

      self.save_hyperparameters(__lowerCAmelCase )
      _a				:							List[Any]							       =      0
      _a				:							Dict							       =      Path(self.hparams.output_dir )
      _a				:							Tuple							       =      self.hparams.cache_dir if self.hparams.cache_dir else None
      if config is None:
          _a				:							Optional[Any]							       =      AutoConfig.from_pretrained(
              self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path		,     **({"""num_labels""": num_labels} if num_labels is not None else {})		,     cache_dir=__lowerCAmelCase		,     **__lowerCAmelCase		,     )
      else:
          _a				:							PretrainedConfig							       =      config

      _a				:							Optional[Any]							       =      ("""encoder_layerdrop""", """decoder_layerdrop""", """dropout""", """attention_dropout""")
      for p in extra_model_params:
          if getattr(self.hparams		,     __lowerCAmelCase		,     __lowerCAmelCase ):
              assert hasattr(self.config		,     __lowerCAmelCase ), f'''model config doesn\'t have a `{p}` attribute'''
              setattr(self.config		,     __lowerCAmelCase		,     getattr(self.hparams		,     __lowerCAmelCase ) )

      if tokenizer is None:
          _a				:							Union[str, Any]							       =      AutoTokenizer.from_pretrained(
              self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path		,     cache_dir=__lowerCAmelCase		,     )
      else:
          _a				:							PreTrainedTokenizer							       =      tokenizer
      _a				:							Union[str, Any]							       =      MODEL_MODES[mode]
      if model is None:
          _a				:							Optional[int]							       =      self.model_type.from_pretrained(
              self.hparams.model_name_or_path		,     from_tf=bool(""".ckpt""" in self.hparams.model_name_or_path )		,     config=self.config		,     cache_dir=__lowerCAmelCase		,     )
      else:
          _a				:							str							       =      model



  def     a_							(			self		,     *__lowerCAmelCase		,     **__lowerCAmelCase ):







      '''simple docstring'''


      _a				:							int							       =      self.model_type.from_pretrained(*__lowerCAmelCase		,     **__lowerCAmelCase )



  def     a_							(			self ):







      '''simple docstring'''


      _a				:							Optional[int]							       =      arg_to_scheduler[self.hparams.lr_scheduler]
      _a				:							Union[str, Any]							       =      get_schedule_func(
          self.opt		,     num_warmup_steps=self.hparams.warmup_steps		,     num_training_steps=self.total_steps() )
      _a				:							List[Any]							       =      {"""scheduler""": scheduler, """interval""": """step""", """frequency""": 1}
      return scheduler



  def     a_							(			self ):







      '''simple docstring'''


      _a				:							List[Any]							       =      self.model
      _a				:							Tuple							       =      ["""bias""", """LayerNorm.weight"""]
      _a				:							Union[str, Any]							       =      [
          {
              """params""": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay )],
              """weight_decay""": self.hparams.weight_decay,
          },
          {
              """params""": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay )],
              """weight_decay""": 0.0,
          },
      ]
      if self.hparams.adafactor:
          _a				:							Union[str, Any]							       =      Adafactor(
              __lowerCAmelCase		,     lr=self.hparams.learning_rate		,     scale_parameter=__lowerCAmelCase		,     relative_step=__lowerCAmelCase )

      else:
          _a				:							List[str]							       =      AdamW(
              __lowerCAmelCase		,     lr=self.hparams.learning_rate		,     eps=self.hparams.adam_epsilon )
      _a				:							List[str]							       =      optimizer

      _a				:							Optional[Any]							       =      self.get_lr_scheduler()

      return [optimizer], [scheduler]



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      return self.validation_step(__lowerCAmelCase		,     __lowerCAmelCase )



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      return self.validation_end(__lowerCAmelCase )



  def     a_							(			self ):







      '''simple docstring'''


      _a				:							int							       =      max(1		,     self.hparams.gpus )  # TODO: consider num_tpu_cores
      _a				:							str							       =      self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
      return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      if mode == "test":
          _a				:							int							       =      len(self.test_dataloader().dataset )
      else:
          _a				:							int							       =      self.get_dataloader("""train"""		,     self.hparams.train_batch_size		,     shuffle=__lowerCAmelCase )
          _a				:							List[str]							       =      len(self.train_dataloader().dataset )



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase = False ):







      '''simple docstring'''


      raise NotImplementedError("""You must implement this for your task""" )



  def     a_							(			self ):







      '''simple docstring'''


      return self.train_loader



  def     a_							(			self ):







      '''simple docstring'''


      return self.get_dataloader("""dev"""		,     self.hparams.eval_batch_size		,     shuffle=__lowerCAmelCase )



  def     a_							(			self ):







      '''simple docstring'''


      return self.get_dataloader("""test"""		,     self.hparams.eval_batch_size		,     shuffle=__lowerCAmelCase )



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      return os.path.join(
          self.hparams.data_dir		,     """cached_{}_{}_{}""".format(
              __lowerCAmelCase		,     list(filter(__lowerCAmelCase		,     self.hparams.model_name_or_path.split("""/""" ) ) ).pop()		,     str(self.hparams.max_seq_length )		,     )		,     )



  @pl.utilities.rank_zero_only
  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							Tuple							       =      self.output_dir.joinpath("""best_tfmr""" )
      _a				:							Optional[int]							       =      self.step_count
      self.model.save_pretrained(__lowerCAmelCase )
      self.tokenizer.save_pretrained(__lowerCAmelCase )



  @staticmethod
  def     a_							(			__lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      parser.add_argument(
          """--model_name_or_path"""		,     default=__lowerCAmelCase		,     type=__lowerCAmelCase		,     required=__lowerCAmelCase		,     help="""Path to pretrained model or model identifier from huggingface.co/models"""		,     )
      parser.add_argument(
          """--config_name"""		,     default=""""""		,     type=__lowerCAmelCase		,     help="""Pretrained config name or path if not the same as model_name""" )
      parser.add_argument(
          """--tokenizer_name"""		,     default=__lowerCAmelCase		,     type=__lowerCAmelCase		,     help="""Pretrained tokenizer name or path if not the same as model_name"""		,     )
      parser.add_argument(
          """--cache_dir"""		,     default=""""""		,     type=__lowerCAmelCase		,     help="""Where do you want to store the pre-trained models downloaded from huggingface.co"""		,     )
      parser.add_argument(
          """--encoder_layerdrop"""		,     type=__lowerCAmelCase		,     help="""Encoder layer dropout probability (Optional). Goes into model.config"""		,     )
      parser.add_argument(
          """--decoder_layerdrop"""		,     type=__lowerCAmelCase		,     help="""Decoder layer dropout probability (Optional). Goes into model.config"""		,     )
      parser.add_argument(
          """--dropout"""		,     type=__lowerCAmelCase		,     help="""Dropout probability (Optional). Goes into model.config"""		,     )
      parser.add_argument(
          """--attention_dropout"""		,     type=__lowerCAmelCase		,     help="""Attention dropout probability (Optional). Goes into model.config"""		,     )
      parser.add_argument("""--learning_rate"""		,     default=5e-5		,     type=__lowerCAmelCase		,     help="""The initial learning rate for Adam.""" )
      parser.add_argument(
          """--lr_scheduler"""		,     default="""linear"""		,     choices=__lowerCAmelCase		,     metavar=__lowerCAmelCase		,     type=__lowerCAmelCase		,     help="""Learning rate scheduler"""		,     )
      parser.add_argument("""--weight_decay"""		,     default=0.0		,     type=__lowerCAmelCase		,     help="""Weight decay if we apply some.""" )
      parser.add_argument("""--adam_epsilon"""		,     default=1e-8		,     type=__lowerCAmelCase		,     help="""Epsilon for Adam optimizer.""" )
      parser.add_argument("""--warmup_steps"""		,     default=0		,     type=__lowerCAmelCase		,     help="""Linear warmup over warmup_steps.""" )
      parser.add_argument("""--num_workers"""		,     default=4		,     type=__lowerCAmelCase		,     help="""kwarg passed to DataLoader""" )
      parser.add_argument("""--num_train_epochs"""		,     dest="""max_epochs"""		,     default=3		,     type=__lowerCAmelCase )
      parser.add_argument("""--train_batch_size"""		,     default=32		,     type=__lowerCAmelCase )
      parser.add_argument("""--eval_batch_size"""		,     default=32		,     type=__lowerCAmelCase )
      parser.add_argument("""--adafactor"""		,     action="""store_true""" )






class      UpperCamelCase       (	pl.Callback       ):



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							Optional[int]							       =      trainer.lr_schedulers[0]["""scheduler"""]
      _a				:							Union[str, Any]							       =      {f'''lr_group_{i}''': lr for i, lr in enumerate(lr_scheduler.get_lr() )}
      pl_module.logger.log_metrics(__lowerCAmelCase )



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      rank_zero_info("""***** Validation results *****""" )
      _a				:							List[str]							       =      trainer.callback_metrics
      # Log results
      for key in sorted(__lowerCAmelCase ):
          if key not in ["log", "progress_bar"]:
              rank_zero_info("""{} = {}\n""".format(__lowerCAmelCase		,     str(metrics[key] ) ) )



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      rank_zero_info("""***** Test results *****""" )
      _a				:							str							       =      trainer.callback_metrics
      # Log and save results to file
      _a				:							Optional[Any]							       =      os.path.join(pl_module.hparams.output_dir		,     """test_results.txt""" )
      with open(__lowerCAmelCase		,     """w""" ) as writer:
          for key in sorted(__lowerCAmelCase ):
              if key not in ["log", "progress_bar"]:
                  rank_zero_info("""{} = {}\n""".format(__lowerCAmelCase		,     str(metrics[key] ) ) )
                  writer.write("""{} = {}\n""".format(__lowerCAmelCase		,     str(metrics[key] ) ) )






def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_						):
    #  To allow all pl args uncomment the following line
    #  parser = pl.Trainer.add_argparse_args(parser)
    parser.add_argument(
        """--output_dir"""   ,	default=UpperCAmelCase_   ,	type=UpperCAmelCase_   ,	required=UpperCAmelCase_   ,	help="""The output directory where the model predictions and checkpoints will be written."""   ,	)
    parser.add_argument(
        """--fp16"""   ,	action="""store_true"""   ,	help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""   ,	)

    parser.add_argument(
        """--fp16_opt_level"""   ,	type=UpperCAmelCase_   ,	default="""O2"""   ,	help=(
            """For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."""
            """See details at https://nvidia.github.io/apex/amp.html"""
        )   ,	)
    parser.add_argument("""--n_tpu_cores"""   ,	dest="""tpu_cores"""   ,	type=UpperCAmelCase_						)
    parser.add_argument("""--max_grad_norm"""   ,	dest="""gradient_clip_val"""   ,	default=1.0   ,	type=UpperCAmelCase_   ,	help="""Max gradient norm"""						)
    parser.add_argument("""--do_train"""   ,	action="""store_true"""   ,	help="""Whether to run training."""						)
    parser.add_argument("""--do_predict"""   ,	action="""store_true"""   ,	help="""Whether to run predictions on the test set."""						)
    parser.add_argument(
        """--gradient_accumulation_steps"""   ,	dest="""accumulate_grad_batches"""   ,	type=UpperCAmelCase_   ,	default=1   ,	help="""Number of updates steps to accumulate before performing a backward/update pass."""   ,	)
    parser.add_argument("""--seed"""   ,	type=UpperCAmelCase_   ,	default=42   ,	help="""random seed for initialization"""						)
    parser.add_argument(
        """--data_dir"""   ,	default=UpperCAmelCase_   ,	type=UpperCAmelCase_   ,	required=UpperCAmelCase_   ,	help="""The input data dir. Should contain the training files for the CoNLL-2003 NER task."""   ,	)





def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_=None   ,	UpperCAmelCase_=True   ,	UpperCAmelCase_=[]   ,	UpperCAmelCase_=None   ,	UpperCAmelCase_=None   ,	**UpperCAmelCase_   ,	):
    pl.seed_everything(args.seed						)

    # init model
    _a				:							List[str]							       =      Path(model.hparams.output_dir						)
    odir.mkdir(exist_ok=UpperCAmelCase_						)

    # add custom checkpoints
    if checkpoint_callback is None:
        _a				:							Dict							       =      pl.callbacks.ModelCheckpoint(
            filepath=args.output_dir   ,	prefix="""checkpoint"""   ,	monitor="""val_loss"""   ,	mode="""min"""   ,	save_top_k=1						)
    if early_stopping_callback:
        extra_callbacks.append(UpperCAmelCase_						)
    if logging_callback is None:
        _a				:							Dict							       =      LoggingCallback()

    _a				:							Optional[int]							       =      {}

    # TODO: remove with PyTorch 1.6 since pl uses native amp
    if args.fpaa:
        _a				:							int							       =      16
        _a				:							Optional[Any]							       =      args.fpaa_opt_level

    if args.gpus > 1:
        _a				:							Dict							       =      """ddp"""

    _a				:							Tuple							       =      args.accumulate_grad_batches
    _a				:							Any							       =      extra_train_kwargs.get("""accelerator"""   ,	UpperCAmelCase_						)
    _a				:							Optional[Any]							       =      extra_train_kwargs.get("""profiler"""   ,	UpperCAmelCase_						)

    _a				:							Optional[Any]							       =      pl.Trainer.from_argparse_args(
        UpperCAmelCase_   ,	weights_summary=UpperCAmelCase_   ,	callbacks=[logging_callback] + extra_callbacks   ,	logger=UpperCAmelCase_   ,	checkpoint_callback=UpperCAmelCase_   ,	**UpperCAmelCase_   ,	)

    if args.do_train:
        trainer.fit(UpperCAmelCase_						)

    return trainer








'''simple docstring'''





import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_reformer import ReformerTokenizer
else:
  a		:	Optional[int]       =      None


a		:	Optional[Any]       =      logging.get_logger(__name__)


a		:	Union[str, Any]       =      '''▁'''

a		:	Any       =      {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

a		:	List[Any]       =      {
    '''vocab_file''': {
        '''google/reformer-crime-and-punishment''': (
            '''https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'''
        )
    },
    '''tokenizer_file''': {
        '''google/reformer-crime-and-punishment''': (
            '''https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json'''
        )
    },
}

a		:	Any       =      {
    '''google/reformer-crime-and-punishment''': 5_2_4_2_8_8,
}




class      UpperCamelCase       (	snake_case__       ):
  a     :str          =					VOCAB_FILES_NAMES
  a     :List[Any]          =					PRETRAINED_VOCAB_FILES_MAP
  a     :Optional[int]          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a     :Any          =					['input_ids', 'attention_mask']
  a     :int          =					ReformerTokenizer



  def __init__(			self		,     __lowerCAmelCase=None		,     __lowerCAmelCase=None		,     __lowerCAmelCase="</s>"		,     __lowerCAmelCase="<unk>"		,     __lowerCAmelCase=[]		,     **__lowerCAmelCase		,     ):







      '''simple docstring'''


      super().__init__(
          __lowerCAmelCase		,     tokenizer_file=__lowerCAmelCase		,     eos_token=__lowerCAmelCase		,     unk_token=__lowerCAmelCase		,     additional_special_tokens=__lowerCAmelCase		,     **__lowerCAmelCase		,     )

      _a				:							List[str]							       =      vocab_file
      _a				:							Optional[int]							       =      False if not self.vocab_file else True



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase = None ):







      '''simple docstring'''


      if not self.can_save_slow_tokenizer:
          raise ValueError(
              """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
              """tokenizer.""" )

      if not os.path.isdir(__lowerCAmelCase ):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
          return
      _a				:							List[str]							       =      os.path.join(
          __lowerCAmelCase		,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

      if os.path.abspath(self.vocab_file ) != os.path.abspath(__lowerCAmelCase ):
          copyfile(self.vocab_file		,     __lowerCAmelCase )

      return (out_vocab_file,)


