



from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase								=      logging.get_logger(__name__)

__lowerCamelCase								=      {
    '''facebook/opt-125m''': '''https://huggingface.co/facebook/opt-125m/blob/main/config.json''',
    '''facebook/opt-350m''': '''https://huggingface.co/facebook/opt-350m/blob/main/config.json''',
    '''facebook/opt-1.3b''': '''https://huggingface.co/facebook/opt-1.3b/blob/main/config.json''',
    '''facebook/opt-2.7b''': '''https://huggingface.co/facebook/opt-2.7b/blob/main/config.json''',
    '''facebook/opt-6.7b''': '''https://huggingface.co/facebook/opt-6.7b/blob/main/config.json''',
    '''facebook/opt-13b''': '''https://huggingface.co/facebook/opt-13b/blob/main/config.json''',
}


class 	snake_case_			(_lowerCAmelCase	):




       """simple docstring"""






       _lowerCamelCase           =					"""opt"""
       _lowerCamelCase           =					["""past_key_values"""]



       def __init__(  self  ,lowercase=50272  ,lowercase=768  ,lowercase=12  ,lowercase=3072  ,lowercase=2048  ,lowercase=True  ,lowercase=False  ,lowercase=None  ,lowercase=0.1  ,lowercase=0.0  ,lowercase=12  ,lowercase="relu"  ,lowercase=0.0  ,lowercase=0.02  ,lowercase=True  ,lowercase=1  ,lowercase=2  ,lowercase=2  ,lowercase=True  ,lowercase=True  ,**lowercase  ,):

             """simple docstring"""





             super().__init__(
                 pad_token_id=lowercase  ,bos_token_id=lowercase  ,eos_token_id=lowercase  ,**lowercase  ,)
             UpperCAmelCase_	:						Optional[int]					   =		vocab_size
             UpperCAmelCase_	:						int					   =		max_position_embeddings
             UpperCAmelCase_	:						Optional[int]					   =		num_attention_heads
             UpperCAmelCase_	:						Tuple					   =		word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
             UpperCAmelCase_	:						Tuple					   =		ffn_dim
             UpperCAmelCase_	:						Optional[int]					   =		hidden_size
             UpperCAmelCase_	:						int					   =		num_hidden_layers
             UpperCAmelCase_	:						int					   =		dropout
             UpperCAmelCase_	:						Optional[Any]					   =		attention_dropout
             UpperCAmelCase_	:						int					   =		activation_function
             UpperCAmelCase_	:						str					   =		init_std
             UpperCAmelCase_	:						Optional[Any]					   =		layerdrop
             UpperCAmelCase_	:						List[Any]					   =		use_cache
             UpperCAmelCase_	:						Any					   =		do_layer_norm_before
             # We keep these variables at `True` for backward compatibility.
             UpperCAmelCase_	:						List[Any]					   =		enable_bias
             UpperCAmelCase_	:						List[str]					   =		layer_norm_elementwise_affine

             # Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
             # with checkpoints that have been fine-tuned before transformers v4.20.1
             # see https://github.com/facebookresearch/metaseq/pull/164
             UpperCAmelCase_	:						Optional[Any]					   =		_remove_final_layer_norm





"""simple docstring"""
import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp
def   A_    (     ):







						'''simple docstring'''


						snake_case_					:str		=     ArgumentParser(
						    description=(
						        """PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"""
						    )      )

						# Optional arguments for the launch helper
						parser.add_argument("""--num_cores""",   type=_lowercase,   default=1,   help="""Number of TPU cores to use (1 or 8)."""      )

						# positional
						parser.add_argument(
						    """training_script""",   type=_lowercase,   help=(
						        """The full path to the single TPU training """
						        """program/script to be launched in parallel, """
						        """followed by all the arguments for the """
						        """training script"""
						    ),   )

						# rest from the training program
						parser.add_argument("""training_script_args""",   nargs=_lowercase      )

						return parser.parse_args()
def   A_    (     ):







						'''simple docstring'''


						snake_case_					:Any		=     parse_args()

						# Import training_script as a module.
						snake_case_					:str		=     Path(args.training_script      )
						sys.path.append(str(script_fpath.parent.resolve()      )      )
						snake_case_					:Dict		=     script_fpath.stem
						snake_case_					:int		=     importlib.import_module(_lowercase      )

						# Patch sys.argv
						snake_case_					:Optional[Any]		=     [args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores      )]

						xmp.spawn(mod._mp_fn,   args=(),   nprocs=args.num_cores      )


if __name__ == "__main__":
						main()
