
"""simple docstring"""
import collections

from .utils import ExplicitEnum, is_torch_available, logging


if is_torch_available():
			import torch


lowercase		:						Union[str, Any] =					logging.get_logger(__name__)




class    _snake_case  :






							def __init__(					self						:			Tuple						, snake_case__						:			Tuple						, snake_case__						:			List[str]=2_1						, snake_case__						:			Tuple=[]						, snake_case__						:			Optional[Any]=None    ):






												"""simple docstring"""



												__a:						str		    =					model
												__a:						List[str]		    =					trace_batch_nums
												__a:						List[str]		    =					abort_after_batch_num

												# keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence
												__a:						Any		    =					collections.deque([]						, __UpperCamelCase    )
												__a:						Optional[Any]		    =					[]
												__a:						List[Any]		    =					0
												__a:						List[Any]		    =					0
												__a:						Tuple		    =					False
												__a:						int		    =					"""                 """

												self.analyse_model()

												self.register_forward_hook()






							def 							__lowercase  (					self						:			int						, snake_case__						:			Dict=None    ):






												"""simple docstring"""



												if frame is not None:
																	self.expand_frame(__UpperCamelCase    )
												self.frames.append("""\n""".join(self.frame    )    )
												__a:						Dict		    =					[]  # start a new frame






							def 							__lowercase  (					self						:			Any						, snake_case__						:			Optional[Any]    ):






												"""simple docstring"""



												self.frame.append(__UpperCamelCase    )






							def 							__lowercase  (					self						:			Dict    ):






												"""simple docstring"""



												print("""\n""".join(self.frames    )    )
												__a:						str		    =					[]






							def 							__lowercase  (					self						:			Optional[Any]    ):






												"""simple docstring"""



												__a:						Dict		    =					[]






							def 							__lowercase  (					self						:			List[str]    ):






												"""simple docstring"""



												print(f"""\nDetected inf/nan during batch_number={self.batch_number}"""    )
												print(f"""Last {len(self.frames    )} forward frames:"""    )
												print(f"""{'abs min':8} {'abs max':8} metadata"""    )
												print("""\n""".join(self.frames    )    )
												print("""\n\n"""    )
												__a:						Union[str, Any]		    =					[]






							def 							__lowercase  (					self						:			Dict    ):






												"""simple docstring"""



												__a:						Optional[Any]		    =					{m: name for name, m in self.model.named_modules()}
												# self.longest_module_name = max(len(v) for v in self.module_names.values())






							def 							__lowercase  (					self						:			Tuple						, snake_case__						:			Optional[Any]						, snake_case__						:			Any    ):






												"""simple docstring"""



												if torch.is_tensor(__UpperCamelCase    ):
																	self.expand_frame(get_abs_min_max(__UpperCamelCase						, __UpperCamelCase    )    )
																	if detect_overflow(__UpperCamelCase						, __UpperCamelCase    ):
																						__a:						int		    =					True
												elif var is None:
																	self.expand_frame(f"""{'None':>17} {ctx}"""    )
												else:
																	self.expand_frame(f"""{'not a tensor':>17} {ctx}"""    )






							def 							__lowercase  (					self						:			List[Any]    ):






												"""simple docstring"""



												self.expand_frame(f"""\n\n{self.prefix} *** Starting batch number={self.batch_number} ***"""    )
												self.expand_frame(f"""{'abs min':8} {'abs max':8} metadata"""    )






							def 							__lowercase  (					self						:			str    ):






												"""simple docstring"""



												self.expand_frame(f"""{self.prefix} *** Finished batch number={self.batch_number-1} ***\n\n"""    )






							def 							__lowercase  (					self						:			Union[str, Any]						, snake_case__						:			Tuple						, snake_case__						:			int						, snake_case__						:			Tuple    ):






												"""simple docstring"""



												self.expand_frame(f"""{self.prefix} {self.module_names[module]} {module.__class__.__name__}"""    )

												# params
												for name, p in module.named_parameters(recurse=__UpperCamelCase    ):
																	self.analyse_variable(__UpperCamelCase						, __UpperCamelCase    )

												# inputs
												if isinstance(__UpperCamelCase						, __UpperCamelCase    ):
																	for i, x in enumerate(__UpperCamelCase    ):
																						self.analyse_variable(__UpperCamelCase						, f"""input[{i}]"""    )
												else:
																	self.analyse_variable(__UpperCamelCase						, """input"""    )

												# outputs
												if isinstance(__UpperCamelCase						, __UpperCamelCase    ):
																	for i, x in enumerate(__UpperCamelCase    ):
																						# possibly a tuple of tuples
																						if isinstance(__UpperCamelCase						, __UpperCamelCase    ):
																											for j, y in enumerate(__UpperCamelCase    ):
																																self.analyse_variable(__UpperCamelCase						, f"""output[{i}][{j}]"""    )
																						else:
																											self.analyse_variable(__UpperCamelCase						, f"""output[{i}]"""    )
												else:
																	self.analyse_variable(__UpperCamelCase						, """output"""    )

												self.save_frame()






							def 							__lowercase  (					self						:			Dict    ):






												"""simple docstring"""



												self.model.apply(self._register_forward_hook    )






							def 							__lowercase  (					self						:			str						, snake_case__						:			List[Any]    ):






												"""simple docstring"""



												module.register_forward_hook(self.forward_hook    )






							def 							__lowercase  (					self						:			str						, snake_case__						:			str						, snake_case__						:			Dict						, snake_case__						:			Union[str, Any]    ):






												"""simple docstring"""



												__a:						List[Any]		    =					False

												__a:						str		    =					True if self.batch_number in self.trace_batch_nums else False
												if trace_mode:
																	self.reset_saved_frames()

												if self.total_calls == 0:
																	self.batch_start_frame()
												self.total_calls += 1

												# count batch numbers - the very first forward hook of the batch will be called when the
												# batch completes - i.e. it gets called very last - we know this batch has finished
												if module == self.model:
																	self.batch_number += 1
																	__a:						int		    =					True

												self.create_frame(__UpperCamelCase						, __UpperCamelCase						, __UpperCamelCase    )

												# if last_frame_of_batch:
												#     self.batch_end_frame()

												if trace_mode:
																	self.trace_frames()

												if last_frame_of_batch:
																	self.batch_start_frame()

												if self.detected_overflow and not trace_mode:
																	self.dump_saved_frames()

																	# now we can abort, as it's pointless to continue running
																	raise ValueError(
																	    """DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. """
																	    """Please scroll up above this traceback to see the activation values prior to this event."""    )

												# abort after certain batch if requested to do so
												if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num:
																	raise ValueError(
																	    f"""DebugUnderflowOverflow: aborting after {self.batch_number} batches due to"""
																	    f""" `abort_after_batch_num={self.abort_after_batch_num}` arg"""    )






def     __UpperCAmelCase    (	lowerCamelCase_	,				lowerCamelCase_ )     ->			Tuple:


					'''simple docstring'''
					__a:						Any		    =					var.abs()
					return f"""{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}"""


def     __UpperCAmelCase    (	lowerCamelCase_	,				lowerCamelCase_ )     ->			str:


					'''simple docstring'''
					__a:						Tuple		    =					False
					if torch.isnan(lowercase_ ).any().item():
										__a:						Union[str, Any]		    =					True
										print(f"""{ctx} has nans""" )
					if torch.isinf(lowercase_ ).any().item():
										__a:						str		    =					True
										print(f"""{ctx} has infs""" )

					# if needed to monitor large elements can enable the following
					if 0:  # and detected:
										__a:						Union[str, Any]		    =					var[torch.ge(var.abs()	,				1_00 )]
										if naaa.numel() > 0:
															print(f"""{ctx}:  n100={naaa.numel()}""" )
										__a:						Optional[Any]		    =					var[torch.ge(var.abs()	,				10_00 )]
										if naaaa.numel() > 0:
															print(f"""{ctx}: n1000={naaaa.numel()}""" )
										__a:						Optional[Any]		    =					var[torch.ge(var.abs()	,				1_00_00 )]
										if naaaaa.numel() > 0:
															print(f"""{ctx}: n10000={naaaaa.numel()}""" )

					if 0:
										print(f"""min={var.min():9.2e} max={var.max():9.2e}""" )

					if 0:
										print(f"""min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})""" )

					return detected




class    _snake_case  (						UpperCAmelCase_       ):
							SCREAMING_SNAKE_CASE		     =						'''underflow_overflow'''
							SCREAMING_SNAKE_CASE		     =						'''tpu_metrics_debug'''


"""simple docstring"""
import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class    _snake_case  (						_lowercase ,				_lowercase ,				_lowercase       ):






							@register_to_config
							def __init__(					self						:			str						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			float						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			int						, snake_case__						:			str						, snake_case__						:			bool = False						, ):






												"""simple docstring"""



												super().__init__()

												__a:						Optional[Any]		    =					nn.Linear(snake_case__						, snake_case__						, bias=snake_case__    )

												__a:						Optional[Any]		    =					nn.Embedding(snake_case__						, snake_case__    )
												__a:						Any		    =					False

												__a:						Any		    =					nn.Dropout(p=snake_case__    )

												__a:						List[str]		    =					TaConfig(
												    d_model=snake_case__						, num_heads=snake_case__						, d_kv=snake_case__						, d_ff=snake_case__						, feed_forward_proj=snake_case__						, dropout_rate=snake_case__						, is_decoder=snake_case__						, is_encoder_decoder=snake_case__						, )
												__a:						Optional[int]		    =					nn.ModuleList()
												for lyr_num in range(snake_case__    ):
																	__a:						Union[str, Any]		    =					TaBlock(snake_case__    )
																	self.encoders.append(snake_case__    )

												__a:						List[str]		    =					TaLayerNorm(snake_case__    )
												__a:						Optional[int]		    =					nn.Dropout(p=snake_case__    )






							def 							__lowercase  (					self						:			str						, snake_case__						:			Optional[Any]						, snake_case__						:			int    ):






												"""simple docstring"""



												__a:						str		    =					self.input_proj(snake_case__    )

												# terminal relative positional encodings
												__a:						Optional[int]		    =					encoder_inputs.shape[1]
												__a:						List[Any]		    =					torch.arange(snake_case__						, device=encoder_inputs.device    )

												__a:						Union[str, Any]		    =					encoder_inputs_mask.sum(-1    )
												__a:						str		    =					torch.roll(input_positions.unsqueeze(0    )						, tuple(seq_lens.tolist()    )						, dims=0    )
												x += self.position_encoding(snake_case__    )

												__a:						Any		    =					self.dropout_pre(snake_case__    )

												# inverted the attention mask
												__a:						Any		    =					encoder_inputs.size()
												__a:						str		    =					self.get_extended_attention_mask(snake_case__						, snake_case__    )

												for lyr in self.encoders:
																	__a:						Any		    =					lyr(snake_case__						, snake_case__    )[0]
												__a:						int		    =					self.layer_norm(snake_case__    )

												return self.dropout_post(snake_case__    ), encoder_inputs_mask
