

'''simple docstring'''





from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


_lowerCAmelCase     :			str    =logging.get_logger(__name__)

_lowerCAmelCase     :			Optional[Any]    ={
    """gpt2""": """https://huggingface.co/gpt2/resolve/main/config.json""",
    """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/config.json""",
    """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/config.json""",
    """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/config.json""",
    """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/config.json""",
}

class __UpperCamelCase    (							_a		):






					'''simple docstring'''



					__magic_name__     			=      "gpt2"
					__magic_name__     			=      ["past_key_values"]
					__magic_name__     			=      {
					    "hidden_size": "n_embd",
					    "max_position_embeddings": "n_positions",
					    "num_attention_heads": "n_head",
					    "num_hidden_layers": "n_layer",
					}





					def __init__(							self							,			lowerCamelCase__=5_0_2_5_7							,			lowerCamelCase__=1_0_2_4							,			lowerCamelCase__=7_6_8							,			lowerCamelCase__=1_2							,			lowerCamelCase__=1_2							,			lowerCamelCase__=None							,			lowerCamelCase__="gelu_new"							,			lowerCamelCase__=0.1							,			lowerCamelCase__=0.1							,			lowerCamelCase__=0.1							,			lowerCamelCase__=1e-5							,			lowerCamelCase__=0.02							,			lowerCamelCase__="cls_index"							,			lowerCamelCase__=True							,			lowerCamelCase__=None							,			lowerCamelCase__=True							,			lowerCamelCase__=0.1							,			lowerCamelCase__=True							,			lowerCamelCase__=True							,			lowerCamelCase__=5_0_2_5_6							,			lowerCamelCase__=5_0_2_5_6							,			lowerCamelCase__=False							,			lowerCamelCase__=False							,			**lowerCamelCase__							,			):
						UpperCAmelCase__:  Dict		       =   vocab_size
						UpperCAmelCase__:  List[str]		       =   n_positions
						UpperCAmelCase__:  Optional[int]		       =   n_embd
						UpperCAmelCase__:  Tuple		       =   n_layer
						UpperCAmelCase__:  Dict		       =   n_head
						UpperCAmelCase__:  int		       =   n_inner
						UpperCAmelCase__:  Dict		       =   activation_function
						UpperCAmelCase__:  Optional[Any]		       =   resid_pdrop
						UpperCAmelCase__:  List[str]		       =   embd_pdrop
						UpperCAmelCase__:  Union[str, Any]		       =   attn_pdrop
						UpperCAmelCase__:  Tuple		       =   layer_norm_epsilon
						UpperCAmelCase__:  Any		       =   initializer_range
						UpperCAmelCase__:  str		       =   summary_type
						UpperCAmelCase__:  Union[str, Any]		       =   summary_use_proj
						UpperCAmelCase__:  Union[str, Any]		       =   summary_activation
						UpperCAmelCase__:  Dict		       =   summary_first_dropout
						UpperCAmelCase__:  List[str]		       =   summary_proj_to_labels
						UpperCAmelCase__:  Optional[Any]		       =   scale_attn_weights
						UpperCAmelCase__:  Optional[int]		       =   use_cache
						UpperCAmelCase__:  List[Any]		       =   scale_attn_by_inverse_layer_idx
						UpperCAmelCase__:  Dict		       =   reorder_and_upcast_attn

						UpperCAmelCase__:  Any		       =   bos_token_id
						UpperCAmelCase__:  Tuple		       =   eos_token_id

						super().__init__(bos_token_id=lowerCamelCase__							,			eos_token_id=lowerCamelCase__							,			**lowerCamelCase__	)

class __UpperCamelCase    (							_a		):






					'''simple docstring'''



					def __init__(							self							,			lowerCamelCase__							,			lowerCamelCase__ = "default"							,			lowerCamelCase__ = None							,			lowerCamelCase__ = False							,			):
						super().__init__(lowerCamelCase__							,			task=lowerCamelCase__							,			patching_specs=lowerCamelCase__							,			use_past=lowerCamelCase__	)
						if not getattr(self._config							,			"pad_token_id"							,			lowerCamelCase__	):
							# TODO: how to do that better?
							UpperCAmelCase__:  Union[str, Any]		       =   0





					@property
					def       _UpperCAmelCase						(							self	):
						UpperCAmelCase__:  Optional[Any]		       =   OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}	)
						if self.use_past:
							self.fill_with_past_key_values_(lowerCamelCase__							,			direction="inputs"	)
							UpperCAmelCase__:  Dict		       =   {0: "batch", 1: "past_sequence + sequence"}
						else:
							UpperCAmelCase__:  Tuple		       =   {0: "batch", 1: "sequence"}

						return common_inputs





					@property
					def       _UpperCAmelCase						(							self	):
						return self._config.n_layer





					@property
					def       _UpperCAmelCase						(							self	):
						return self._config.n_head





					def       _UpperCAmelCase						(							self							,			lowerCamelCase__							,			lowerCamelCase__ = -1							,			lowerCamelCase__ = -1							,			lowerCamelCase__ = False							,			lowerCamelCase__ = None							,			):
						UpperCAmelCase__:  List[Any]		       =   super(lowerCamelCase__							,			self	).generate_dummy_inputs(
						    lowerCamelCase__							,			batch_size=lowerCamelCase__							,			seq_length=lowerCamelCase__							,			is_pair=lowerCamelCase__							,			framework=lowerCamelCase__	)

						# We need to order the input in the way they appears in the forward()
						UpperCAmelCase__:  Tuple		       =   OrderedDict({"input_ids": common_inputs["input_ids"]}	)

						# Need to add the past_keys
						if self.use_past:
							if not is_torch_available():
								raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."	)
							else:
								import torch

								UpperCAmelCase__:  Optional[Any]		       =   common_inputs["input_ids"].shape
								# Not using the same length for past_key_values
								UpperCAmelCase__:  Tuple		       =   seqlen + 2
								UpperCAmelCase__:  Dict		       =   (
								    batch,
								    self.num_attention_heads,
								    past_key_values_length,
								    self._config.hidden_size // self.num_attention_heads,
								)
								UpperCAmelCase__:  Optional[Any]		       =   [
								    (torch.zeros(lowerCamelCase__	), torch.zeros(lowerCamelCase__	)) for _ in range(self.num_layers	)
								]

						UpperCAmelCase__:  Union[str, Any]		       =   common_inputs["attention_mask"]
						if self.use_past:
							UpperCAmelCase__:  List[Any]		       =   ordered_inputs["attention_mask"].dtype
							UpperCAmelCase__:  Dict		       =   torch.cat(
							    [ordered_inputs["attention_mask"], torch.ones(lowerCamelCase__							,			lowerCamelCase__							,			dtype=lowerCamelCase__	)]							,			dim=1	)

						return ordered_inputs





					@property
					def       _UpperCAmelCase						(							self	):
						return 1_3


import numpy as np



def  _A  (			SCREAMING_SNAKE_CASE			):
	UpperCAmelCase__:  List[Any]		       =   np.exp(SCREAMING_SNAKE_CASE			)

	# Add up the all the exponentials
	UpperCAmelCase__:  str		       =   np.sum(SCREAMING_SNAKE_CASE			)

	# Divide every exponent by the sum of all exponents
	UpperCAmelCase__:  Tuple		       =   exponent_vector / sum_of_exponents

	return softmax_vector


if __name__ == "__main__":
					print(softmax((0,)))