



'''simple docstring'''



import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


a__						:			List[str]										=			logging.getLogger(__name__)

require_version('pytorch_lightning>=1.0.4')

a__						:			Any										=			{
    'base': AutoModel,
    'sequence-classification': AutoModelForSequenceClassification,
    'question-answering': AutoModelForQuestionAnswering,
    'pretraining': AutoModelForPreTraining,
    'token-classification': AutoModelForTokenClassification,
    'language-modeling': AutoModelWithLMHead,
    'summarization': AutoModelForSeqaSeqLM,
    'translation': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
a__						:			List[str]										=			{
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
a__						:			List[Any]										=			sorted(arg_to_scheduler.keys())
a__						:			Optional[Any]										=			'{' + ', '.join(arg_to_scheduler_choices) + '}'
class        UpperCAmelCase__  (    pl.LightningModule):



 def __init__(  self		,	lowercase		,	lowercase=None		,	lowercase="base"		,	lowercase=None		,	lowercase=None		,	lowercase=None		,	**lowercase		,	)  ->							Tuple:
  super().__init__()
  # TODO: move to self.save_hyperparameters()
  # self.save_hyperparameters()
  # can also expand arguments into trainer signature for easier reading

  self.save_hyperparameters(lowercase			)
  __UpperCamelCase				      =  0
  __UpperCamelCase				      =  Path(self.hparams.output_dir			)
  __UpperCamelCase				      =  self.hparams.cache_dir if self.hparams.cache_dir else None
  if config is None:
   __UpperCamelCase				      =  AutoConfig.from_pretrained(
       self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path		,	**({"""num_labels""": num_labels} if num_labels is not None else {})		,	cache_dir=lowercase		,	**lowercase		,	)
  else:
   __UpperCamelCase				      =  config

  __UpperCamelCase				      =  ("""encoder_layerdrop""", """decoder_layerdrop""", """dropout""", """attention_dropout""")
  for p in extra_model_params:
   if getattr(self.hparams		,	lowercase		,	lowercase			):
    assert hasattr(self.config		,	lowercase			), f"model config doesn't have a `{p}` attribute"
    setattr(self.config		,	lowercase		,	getattr(self.hparams		,	lowercase			)			)

  if tokenizer is None:
   __UpperCamelCase				      =  AutoTokenizer.from_pretrained(
       self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path		,	cache_dir=lowercase		,	)
  else:
   __UpperCamelCase				      =  tokenizer
  __UpperCamelCase				      =  MODEL_MODES[mode]
  if model is None:
   __UpperCamelCase				      =  self.model_type.from_pretrained(
       self.hparams.model_name_or_path		,	from_tf=bool(""".ckpt""" in self.hparams.model_name_or_path			)		,	config=self.config		,	cache_dir=lowercase		,	)
  else:
   __UpperCamelCase				      =  model



 def    __lowerCamelCase				(  self		,	*lowercase		,	**lowercase			)  ->							Union[str, Any]:
  __UpperCamelCase				      =  self.model_type.from_pretrained(*lowercase		,	**lowercase			)



 def    __lowerCamelCase				(  self			)  ->							int:
  __UpperCamelCase				      =  arg_to_scheduler[self.hparams.lr_scheduler]
  __UpperCamelCase				      =  get_schedule_func(
      self.opt		,	num_warmup_steps=self.hparams.warmup_steps		,	num_training_steps=self.total_steps()			)
  __UpperCamelCase				      =  {"""scheduler""": scheduler, """interval""": """step""", """frequency""": 1}
  return scheduler



 def    __lowerCamelCase				(  self			)  ->							Union[str, Any]:
  __UpperCamelCase				      =  self.model
  __UpperCamelCase				      =  ["""bias""", """LayerNorm.weight"""]
  __UpperCamelCase				      =  [
      {
          """params""": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay			)],
          """weight_decay""": self.hparams.weight_decay,
      },
      {
          """params""": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay			)],
          """weight_decay""": 0.0,
      },
  ]
  if self.hparams.adafactor:
   __UpperCamelCase				      =  Adafactor(
       lowercase		,	lr=self.hparams.learning_rate		,	scale_parameter=lowercase		,	relative_step=lowercase			)

  else:
   __UpperCamelCase				      =  AdamW(
       lowercase		,	lr=self.hparams.learning_rate		,	eps=self.hparams.adam_epsilon			)
  __UpperCamelCase				      =  optimizer

  __UpperCamelCase				      =  self.get_lr_scheduler()

  return [optimizer], [scheduler]



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase			)  ->							Optional[Any]:
  return self.validation_step(lowercase		,	lowercase			)



 def    __lowerCamelCase				(  self		,	lowercase			)  ->							List[Any]:
  return self.validation_end(lowercase			)



 def    __lowerCamelCase				(  self			)  ->							int:
  __UpperCamelCase				      =  max(1		,	self.hparams.gpus			)  # TODO: consider num_tpu_cores
  __UpperCamelCase				      =  self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
  return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs



 def    __lowerCamelCase				(  self		,	lowercase			)  ->							Union[str, Any]:
  if mode == "test":
   __UpperCamelCase				      =  len(self.test_dataloader().dataset			)
  else:
   __UpperCamelCase				      =  self.get_dataloader("""train"""		,	self.hparams.train_batch_size		,	shuffle=lowercase			)
   __UpperCamelCase				      =  len(self.train_dataloader().dataset			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase = False			)  ->							Optional[Any]:
  raise NotImplementedError("""You must implement this for your task"""			)



 def    __lowerCamelCase				(  self			)  ->							Dict:
  return self.train_loader



 def    __lowerCamelCase				(  self			)  ->							Optional[int]:
  return self.get_dataloader("""dev"""		,	self.hparams.eval_batch_size		,	shuffle=lowercase			)



 def    __lowerCamelCase				(  self			)  ->							List[str]:
  return self.get_dataloader("""test"""		,	self.hparams.eval_batch_size		,	shuffle=lowercase			)



 def    __lowerCamelCase				(  self		,	lowercase			)  ->							Optional[int]:
  return os.path.join(
      self.hparams.data_dir		,	"""cached_{}_{}_{}""".format(
          lowercase		,	list(filter(lowercase		,	self.hparams.model_name_or_path.split("""/"""			)			)			).pop()		,	str(self.hparams.max_seq_length			)		,	)		,	)



 @pl.utilities.rank_zero_only
 def    __lowerCamelCase				(  self		,	lowercase			)  ->							None:
  __UpperCamelCase				      =  self.output_dir.joinpath("""best_tfmr"""			)
  __UpperCamelCase				      =  self.step_count
  self.model.save_pretrained(lowercase			)
  self.tokenizer.save_pretrained(lowercase			)




 @staticmethod
 def    __lowerCamelCase				(  lowercase		,	lowercase			)  ->							int:
  parser.add_argument(
      """--model_name_or_path"""		,	default=lowercase		,	type=lowercase		,	required=lowercase		,	help="""Path to pretrained model or model identifier from huggingface.co/models"""		,	)
  parser.add_argument(
      """--config_name"""		,	default=""""""		,	type=lowercase		,	help="""Pretrained config name or path if not the same as model_name"""			)
  parser.add_argument(
      """--tokenizer_name"""		,	default=lowercase		,	type=lowercase		,	help="""Pretrained tokenizer name or path if not the same as model_name"""		,	)
  parser.add_argument(
      """--cache_dir"""		,	default=""""""		,	type=lowercase		,	help="""Where do you want to store the pre-trained models downloaded from huggingface.co"""		,	)
  parser.add_argument(
      """--encoder_layerdrop"""		,	type=lowercase		,	help="""Encoder layer dropout probability (Optional). Goes into model.config"""		,	)
  parser.add_argument(
      """--decoder_layerdrop"""		,	type=lowercase		,	help="""Decoder layer dropout probability (Optional). Goes into model.config"""		,	)
  parser.add_argument(
      """--dropout"""		,	type=lowercase		,	help="""Dropout probability (Optional). Goes into model.config"""		,	)
  parser.add_argument(
      """--attention_dropout"""		,	type=lowercase		,	help="""Attention dropout probability (Optional). Goes into model.config"""		,	)
  parser.add_argument("""--learning_rate"""		,	default=5E-5		,	type=lowercase		,	help="""The initial learning rate for Adam."""			)
  parser.add_argument(
      """--lr_scheduler"""		,	default="""linear"""		,	choices=lowercase		,	metavar=lowercase		,	type=lowercase		,	help="""Learning rate scheduler"""		,	)
  parser.add_argument("""--weight_decay"""		,	default=0.0		,	type=lowercase		,	help="""Weight decay if we apply some."""			)
  parser.add_argument("""--adam_epsilon"""		,	default=1E-8		,	type=lowercase		,	help="""Epsilon for Adam optimizer."""			)
  parser.add_argument("""--warmup_steps"""		,	default=0		,	type=lowercase		,	help="""Linear warmup over warmup_steps."""			)
  parser.add_argument("""--num_workers"""		,	default=4		,	type=lowercase		,	help="""kwarg passed to DataLoader"""			)
  parser.add_argument("""--num_train_epochs"""		,	dest="""max_epochs"""		,	default=3		,	type=lowercase			)
  parser.add_argument("""--train_batch_size"""		,	default=3_2		,	type=lowercase			)
  parser.add_argument("""--eval_batch_size"""		,	default=3_2		,	type=lowercase			)
  parser.add_argument("""--adafactor"""		,	action="""store_true"""			)


class        UpperCAmelCase__  (    pl.Callback):



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase			)  ->							Any:
  __UpperCamelCase				      =  trainer.lr_schedulers[0]["""scheduler"""]
  __UpperCamelCase				      =  {f"lr_group_{i}": lr for i, lr in enumerate(lr_scheduler.get_lr()			)}
  pl_module.logger.log_metrics(lowercase			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase			)  ->							str:
  rank_zero_info("""***** Validation results *****"""			)
  __UpperCamelCase				      =  trainer.callback_metrics
  # Log results
  for key in sorted(lowercase			):
   if key not in ["log", "progress_bar"]:
    rank_zero_info("""{} = {}\n""".format(lowercase		,	str(metrics[key]			)			)			)




 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase			)  ->							Optional[Any]:
  rank_zero_info("""***** Test results *****"""			)
  __UpperCamelCase				      =  trainer.callback_metrics
  # Log and save results to file
  __UpperCamelCase				      =  os.path.join(pl_module.hparams.output_dir		,	"""test_results.txt"""			)
  with open(lowercase		,	"""w"""			) as writer:
   for key in sorted(lowercase			):
    if key not in ["log", "progress_bar"]:
     rank_zero_info("""{} = {}\n""".format(lowercase		,	str(metrics[key]			)			)			)
     writer.write("""{} = {}\n""".format(lowercase		,	str(metrics[key]			)			)			)







def _lowercase   (  __A							,__A       ):



 '''simple docstring'''



 parser.add_argument(
     """--output_dir"""							,default=__A							,type=__A							,required=__A							,help="""The output directory where the model predictions and checkpoints will be written."""							,)
 parser.add_argument(
     """--fp16"""							,action="""store_true"""							,help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""							,)

 parser.add_argument(
     """--fp16_opt_level"""							,type=__A							,default="""O2"""							,help=(
         """For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."""
         """See details at https://nvidia.github.io/apex/amp.html"""
     )							,)
 parser.add_argument("""--n_tpu_cores"""							,dest="""tpu_cores"""							,type=__A       )
 parser.add_argument("""--max_grad_norm"""							,dest="""gradient_clip_val"""							,default=1.0							,type=__A							,help="""Max gradient norm"""       )
 parser.add_argument("""--do_train"""							,action="""store_true"""							,help="""Whether to run training."""       )
 parser.add_argument("""--do_predict"""							,action="""store_true"""							,help="""Whether to run predictions on the test set."""       )
 parser.add_argument(
     """--gradient_accumulation_steps"""							,dest="""accumulate_grad_batches"""							,type=__A							,default=1							,help="""Number of updates steps to accumulate before performing a backward/update pass."""							,)
 parser.add_argument("""--seed"""							,type=__A							,default=42							,help="""random seed for initialization"""       )
 parser.add_argument(
     """--data_dir"""							,default=__A							,type=__A							,required=__A							,help="""The input data dir. Should contain the training files for the CoNLL-2003 NER task."""							,)







def _lowercase   (  __A							,__A							,__A=None							,__A=True							,__A=[]							,__A=None							,__A=None							,**__A							,):



 '''simple docstring'''



 pl.seed_everything(args.seed       )

 # init model
 __UpperCamelCase				      =  Path(model.hparams.output_dir       )
 odir.mkdir(exist_ok=__A       )

 # add custom checkpoints
 if checkpoint_callback is None:
  __UpperCamelCase				      =  pl.callbacks.ModelCheckpoint(
      filepath=args.output_dir							,prefix="""checkpoint"""							,monitor="""val_loss"""							,mode="""min"""							,save_top_k=1       )
 if early_stopping_callback:
  extra_callbacks.append(__A       )
 if logging_callback is None:
  __UpperCamelCase				      =  LoggingCallback()

 __UpperCamelCase				      =  {}

 # TODO: remove with PyTorch 1.6 since pl uses native amp
 if args.fpaa:
  __UpperCamelCase				      =  16
  __UpperCamelCase				      =  args.fpaa_opt_level

 if args.gpus > 1:
  __UpperCamelCase				      =  """ddp"""

 __UpperCamelCase				      =  args.accumulate_grad_batches
 __UpperCamelCase				      =  extra_train_kwargs.get("""accelerator"""							,__A       )
 __UpperCamelCase				      =  extra_train_kwargs.get("""profiler"""							,__A       )

 __UpperCamelCase				      =  pl.Trainer.from_argparse_args(
     __A							,weights_summary=__A							,callbacks=[logging_callback] + extra_callbacks							,logger=__A							,checkpoint_callback=__A							,**__A							,)

 if args.do_train:
  trainer.fit(__A       )

 return trainer










'''simple docstring'''



import argparse

from transformers import TaConfig, TaForConditionalGeneration, load_tf_weights_in_ta
from transformers.utils import logging


logging.set_verbosity_info()







def _lowercase   (  __A							,__A							,__A       ):



 '''simple docstring'''



 __UpperCamelCase				      =  TaConfig.from_json_file(__A       )
 print(f"Building PyTorch model from configuration: {config}"       )
 __UpperCamelCase				      =  TaForConditionalGeneration(__A       )

 # Load weights from tf checkpoint
 load_tf_weights_in_ta(__A							,__A							,__A       )

 # Save pytorch-model
 print(f"Save PyTorch model to {pytorch_dump_path}"       )
 model.save_pretrained(__A       )


if __name__ == "__main__":
 a__						:			int										=			argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
 )
 parser.add_argument(
     '--config_file',
     default=None,
     type=str,
     required=True,
     help=(
         'The config json file corresponding to the pre-trained T5 model. \nThis specifies the model architecture.'
     ),
 )
 parser.add_argument(
     '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
 )
 a__						:			Optional[Any]										=			parser.parse_args()
 convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path)





