
"""simple docstring"""





import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
    import transformer_engine.pytorch as te







def       __a	(						__lowerCamelCase,						__lowerCamelCase=True,						__lowerCamelCase=True,						__lowerCamelCase=True    ):
   if not is_fpa_available():
      raise ImportError("Using `convert_model` requires transformer_engine to be installed."    )
   for name, module in model.named_children():
      if isinstance(__lowerCamelCase,						nn.Linear    ) and to_transformer_engine and _convert_linear:
         # Return early if the linear layer weights are not multiples of 16
         if any(p % 16 != 0 for p in module.weight.shape    ):
            return
         UpperCAmelCase_   :   List[str]     	=       module.bias is not None
         UpperCAmelCase_   :   int     	=       te.Linear(
             module.in_features,						module.out_features,						bias=__lowerCamelCase,						params_dtype=module.weight.dtype    )
         UpperCAmelCase_   :   Dict     	=       module.weight.data.clone()
         if has_bias:
            UpperCAmelCase_   :   Dict     	=       module.bias.data.clone()

         setattr(__lowerCamelCase,						__lowerCamelCase,						__lowerCamelCase    )
      elif isinstance(__lowerCamelCase,						nn.LayerNorm    ) and to_transformer_engine and _convert_ln:
         UpperCAmelCase_   :   Any     	=       te.LayerNorm(module.normalized_shape[0],						eps=module.eps,						params_dtype=module.weight.dtype    )
         UpperCAmelCase_   :   int     	=       module.weight.data.clone()
         UpperCAmelCase_   :   List[str]     	=       module.bias.data.clone()

         setattr(__lowerCamelCase,						__lowerCamelCase,						__lowerCamelCase    )
      elif isinstance(__lowerCamelCase,						te.Linear    ) and not to_transformer_engine and _convert_linear:
         UpperCAmelCase_   :   Any     	=       module.bias is not None
         UpperCAmelCase_   :   Any     	=       nn.Linear(
             module.in_features,						module.out_features,						bias=__lowerCamelCase,						params_dtype=module.weight.dtype    )
         UpperCAmelCase_   :   Optional[int]     	=       module.weight.data.clone()
         if has_bias:
            UpperCAmelCase_   :   Any     	=       module.bias.data.clone()

         setattr(__lowerCamelCase,						__lowerCamelCase,						__lowerCamelCase    )
      elif isinstance(__lowerCamelCase,						te.LayerNorm    ) and not to_transformer_engine and _convert_ln:
         UpperCAmelCase_   :   Tuple     	=       nn.LayerNorm(module.normalized_shape[0],						eps=module.eps,						params_dtype=module.weight.dtype    )
         UpperCAmelCase_   :   Optional[Any]     	=       module.weight.data.clone()
         UpperCAmelCase_   :   Dict     	=       module.bias.data.clone()

         setattr(__lowerCamelCase,						__lowerCamelCase,						__lowerCamelCase    )
      else:
         convert_model(
             __lowerCamelCase,						to_transformer_engine=__lowerCamelCase,						_convert_linear=_convert_linear,						_convert_ln=_convert_ln,						)







def       __a	(						__lowerCamelCase    ):
   if not is_fpa_available():
      raise ImportError("Using `has_transformer_engine_layers` requires transformer_engine to be installed."    )
   for m in model.modules():
      if isinstance(__lowerCamelCase,						(te.LayerNorm, te.Linear)    ):
         return True
   return False




from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A							=       logging.get_logger(__name__)

_A							=       {
    '''facebook/opt-125m''': '''https://huggingface.co/facebook/opt-125m/blob/main/config.json''',
    '''facebook/opt-350m''': '''https://huggingface.co/facebook/opt-350m/blob/main/config.json''',
    '''facebook/opt-1.3b''': '''https://huggingface.co/facebook/opt-1.3b/blob/main/config.json''',
    '''facebook/opt-2.7b''': '''https://huggingface.co/facebook/opt-2.7b/blob/main/config.json''',
    '''facebook/opt-6.7b''': '''https://huggingface.co/facebook/opt-6.7b/blob/main/config.json''',
    '''facebook/opt-13b''': '''https://huggingface.co/facebook/opt-13b/blob/main/config.json''',
}

class 		lowercase_							(							__SCREAMING_SNAKE_CASE					):
 A__    :					Dict    =     """opt"""
 A__    :					int    =     ["""past_key_values"""]

 def __init__(   self  ,						__UpperCamelCase=5_0_2_7_2  ,						__UpperCamelCase=7_6_8  ,						__UpperCamelCase=1_2  ,						__UpperCamelCase=3_0_7_2  ,						__UpperCamelCase=2_0_4_8  ,						__UpperCamelCase=True  ,						__UpperCamelCase=False  ,						__UpperCamelCase=None  ,						__UpperCamelCase=0.1  ,						__UpperCamelCase=0.0  ,						__UpperCamelCase=1_2  ,						__UpperCamelCase="relu"  ,						__UpperCamelCase=0.0  ,						__UpperCamelCase=0.02  ,						__UpperCamelCase=True  ,						__UpperCamelCase=1  ,						__UpperCamelCase=2  ,						__UpperCamelCase=2  ,						__UpperCamelCase=True  ,						__UpperCamelCase=True  ,						**__UpperCamelCase  ,						):




        """simple docstring"""


        super().__init__(
            pad_token_id=__UpperCamelCase  ,						bos_token_id=__UpperCamelCase  ,						eos_token_id=__UpperCamelCase  ,						**__UpperCamelCase  ,						)
        UpperCamelCase_											=							vocab_size
        UpperCamelCase_											=							max_position_embeddings
        UpperCamelCase_											=							num_attention_heads
        UpperCamelCase_											=							word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
        UpperCamelCase_											=							ffn_dim
        UpperCamelCase_											=							hidden_size
        UpperCamelCase_											=							num_hidden_layers
        UpperCamelCase_											=							dropout
        UpperCamelCase_											=							attention_dropout
        UpperCamelCase_											=							activation_function
        UpperCamelCase_											=							init_std
        UpperCamelCase_											=							layerdrop
        UpperCamelCase_											=							use_cache
        UpperCamelCase_											=							do_layer_norm_before
        # We keep these variables at `True` for backward compatibility.
        UpperCamelCase_											=							enable_bias
        UpperCamelCase_											=							layer_norm_elementwise_affine

        # Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
        # with checkpoints that have been fine-tuned before transformers v4.20.1
        # see https://github.com/facebookresearch/metaseq/pull/164
        UpperCamelCase_											=							_remove_final_layer_norm



