"""simple docstring"""



from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_UpperCamelCase         =		logging.get_logger(__name__)

_UpperCamelCase         =		{
    "google/bigbird-pegasus-large-arxiv": (
        "https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json"
    ),
    "google/bigbird-pegasus-large-pubmed": (
        "https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json"
    ),
    "google/bigbird-pegasus-large-bigpatent": (
        "https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json"
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}
class 	__UpperCAmelCase						(__A    ):







			'''simple docstring'''
			_UpperCamelCase     :	int				    =							'bigbird_pegasus'
			_UpperCamelCase     :	Optional[int]				    =							['past_key_values']
			_UpperCamelCase     :	Any				    =							{
			    'num_attention_heads': 'encoder_attention_heads',
			    'hidden_size': 'd_model',
			    'attention_probs_dropout_prob': 'attention_dropout',
			}





			def __init__( self      ,			snake_case_=96_103      ,			snake_case_=4_096      ,			snake_case_=16      ,			snake_case_=4_096      ,			snake_case_=16      ,			snake_case_=16      ,			snake_case_=4_096      ,			snake_case_=16      ,			snake_case_=0.0      ,			snake_case_=0.0      ,			snake_case_=True      ,			snake_case_=True      ,			snake_case_="gelu_new"      ,			snake_case_=1_024      ,			snake_case_=0.1      ,			snake_case_=0.0      ,			snake_case_=0.0      ,			snake_case_=0.02      ,			snake_case_=2      ,			snake_case_=0.0      ,			snake_case_=True      ,			snake_case_=0      ,			snake_case_=2      ,			snake_case_=1      ,			snake_case_="block_sparse"      ,			snake_case_=64      ,			snake_case_=3      ,			snake_case_=False      ,			**snake_case_      ,			):







										'''simple docstring'''


										A__					:    Any														= vocab_size
										A__					:    Optional[Any]														= max_position_embeddings
										A__					:    List[Any]														= d_model
										A__					:    Optional[int]														= encoder_ffn_dim
										A__					:    Dict														= encoder_layers
										A__					:    int														= encoder_attention_heads
										A__					:    str														= decoder_ffn_dim
										A__					:    List[str]														= decoder_layers
										A__					:    Optional[int]														= decoder_attention_heads
										A__					:    List[Any]														= dropout
										A__					:    Optional[Any]														= attention_dropout
										A__					:    Optional[int]														= activation_dropout
										A__					:    str														= activation_function
										A__					:    int														= init_std
										A__					:    List[str]														= encoder_layerdrop
										A__					:    Optional[int]														= decoder_layerdrop
										A__					:    Tuple														= classifier_dropout
										A__					:    Optional[int]														= use_cache
										A__					:    Dict														= encoder_layers
										A__					:    int														= scale_embedding  # scale factor will be sqrt(d_model) if True

										# extra config
										A__					:    str														= attention_type
										A__					:    Dict														= block_size
										A__					:    Union[str, Any]														= num_random_blocks
										A__					:    Any														= use_bias

										super().__init__(
										    pad_token_id=snake_case_      ,			bos_token_id=snake_case_      ,			eos_token_id=snake_case_      ,			is_encoder_decoder=snake_case_      ,			decoder_start_token_id=snake_case_      ,			**snake_case_      ,			)





class 	__UpperCAmelCase						(__A    ):







			'''simple docstring'''





			@property
			def 		lowerCamelCase				( self       ):







										'''simple docstring'''


										if self.task in ["default", "seq2seq-lm"]:
																	A__					:    Optional[Any]														= OrderedDict(
																	    [
																	        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
																	        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
																	    ]       )

																	if self.use_past:
																								A__					:    List[Any]														= {0: """batch"""}
																								A__					:    int														= {0: """batch""", 1: """past_decoder_sequence + sequence"""}
																	else:
																								A__					:    Any														= {0: """batch""", 1: """decoder_sequence"""}
																								A__					:    int														= {0: """batch""", 1: """decoder_sequence"""}

																	if self.use_past:
																								self.fill_with_past_key_values_(snake_case_      ,			direction="""inputs"""       )
										elif self.task == "causal-lm":
																	# TODO: figure this case out.
																	A__					:    Optional[Any]														= OrderedDict(
																	    [
																	        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
																	        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
																	    ]       )
																	if self.use_past:
																								A__							,       A__					:    List[str]														= self.num_layers
																								for i in range(snake_case_       ):
																															A__					:    Union[str, Any]														= {0: """batch""", 2: """past_sequence + sequence"""}
																															A__					:    Optional[int]														= {0: """batch""", 2: """past_sequence + sequence"""}
										else:
																	A__					:    str														= OrderedDict(
																	    [
																	        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
																	        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
																	        ("""decoder_input_ids""", {0: """batch""", 1: """decoder_sequence"""}),
																	        ("""decoder_attention_mask""", {0: """batch""", 1: """decoder_sequence"""}),
																	    ]       )

										return common_inputs





			@property
			def 		lowerCamelCase				( self       ):







										'''simple docstring'''


										if self.task in ["default", "seq2seq-lm"]:
																	A__					:    Union[str, Any]														= super().outputs
										else:
																	A__					:    Any														= super(snake_case_      ,			self       ).outputs
																	if self.use_past:
																								A__							,       A__					:    Any														= self.num_layers
																								for i in range(snake_case_       ):
																															A__					:    Dict														= {0: """batch""", 2: """past_sequence + sequence"""}
																															A__					:    int														= {0: """batch""", 2: """past_sequence + sequence"""}
										return common_outputs





			def 		lowerCamelCase				( self      ,			snake_case_      ,			snake_case_ = -1      ,			snake_case_ = -1      ,			snake_case_ = False      ,			snake_case_ = None      ,			):







										'''simple docstring'''


										A__					:    Dict														= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    snake_case_      ,			snake_case_      ,			snake_case_      ,			snake_case_      ,			snake_case_       )

										# Generate decoder inputs
										A__					:    Union[str, Any]														= seq_length if not self.use_past else 1
										A__					:    Any														= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    snake_case_      ,			snake_case_      ,			snake_case_      ,			snake_case_      ,			snake_case_       )
										A__					:    List[str]														= {F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
										A__					:    List[Any]														= dict(**snake_case_      ,			**snake_case_       )

										if self.use_past:
																	if not is_torch_available():
																								raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""       )
																	else:
																								import torch
																	A__							,       A__					:    Optional[int]														= common_inputs["""input_ids"""].shape
																	A__					:    Dict														= common_inputs["""decoder_input_ids"""].shape[1]
																	A__							,       A__					:    Tuple														= self.num_attention_heads
																	A__					:    Tuple														= (
																	    batch,
																	    num_encoder_attention_heads,
																	    encoder_seq_length,
																	    self._config.hidden_size // num_encoder_attention_heads,
																	)
																	A__					:    Union[str, Any]														= decoder_seq_length + 3
																	A__					:    int														= (
																	    batch,
																	    num_decoder_attention_heads,
																	    decoder_past_length,
																	    self._config.hidden_size // num_decoder_attention_heads,
																	)

																	A__					:    List[str]														= torch.cat(
																	    [common_inputs["""decoder_attention_mask"""], torch.ones(snake_case_      ,			snake_case_       )]      ,			dim=1       )

																	A__					:    Union[str, Any]														= []
																	# If the number of encoder and decoder layers are present in the model configuration, both are considered
																	A__							,       A__					:    Optional[Any]														= self.num_layers
																	A__					:    Optional[int]														= min(snake_case_      ,			snake_case_       )
																	A__					:    int														= max(snake_case_      ,			snake_case_       ) - min_num_layers
																	A__					:    Any														= """encoder""" if num_encoder_layers > num_decoder_layers else """decoder"""

																	for _ in range(snake_case_       ):
																								common_inputs["past_key_values"].append(
																								    (
																								        torch.zeros(snake_case_       ),
																								        torch.zeros(snake_case_       ),
																								        torch.zeros(snake_case_       ),
																								        torch.zeros(snake_case_       ),
																								    )       )
																	# TODO: test this.
																	A__					:    str														= encoder_shape if remaining_side_name == """encoder""" else decoder_shape
																	for _ in range(snake_case_      ,			snake_case_       ):
																								common_inputs["past_key_values"].append((torch.zeros(snake_case_       ), torch.zeros(snake_case_       ))       )
										return common_inputs





			def 		lowerCamelCase				( self      ,			snake_case_      ,			snake_case_ = -1      ,			snake_case_ = -1      ,			snake_case_ = False      ,			snake_case_ = None      ,			):







										'''simple docstring'''


										A__					:    int														= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    snake_case_      ,			snake_case_      ,			snake_case_      ,			snake_case_      ,			snake_case_       )

										if self.use_past:
																	if not is_torch_available():
																								raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""       )
																	else:
																								import torch
																	A__							,       A__					:    Dict														= common_inputs["""input_ids"""].shape
																	# Not using the same length for past_key_values
																	A__					:    List[Any]														= seqlen + 2
																	A__							,       A__					:    Any														= self.num_layers
																	A__							,       A__					:    int														= self.num_attention_heads
																	A__					:    List[Any]														= (
																	    batch,
																	    num_encoder_attention_heads,
																	    past_key_values_length,
																	    self._config.hidden_size // num_encoder_attention_heads,
																	)

																	A__					:    str														= common_inputs["""attention_mask"""].dtype
																	A__					:    Dict														= torch.cat(
																	    [common_inputs["""attention_mask"""], torch.ones(snake_case_      ,			snake_case_      ,			dtype=snake_case_       )]      ,			dim=1       )
																	A__					:    List[str]														= [
																	    (torch.zeros(snake_case_       ), torch.zeros(snake_case_       )) for _ in range(snake_case_       )
																	]
										return common_inputs





			def 		lowerCamelCase				( self      ,			snake_case_      ,			snake_case_ = -1      ,			snake_case_ = -1      ,			snake_case_ = False      ,			snake_case_ = None      ,			):







										'''simple docstring'''


										A__					:    List[Any]														= compute_effective_axis_dimension(
										    snake_case_      ,			fixed_dimension=OnnxConfig.default_fixed_batch      ,			num_token_to_add=0       )

										# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
										A__					:    Dict														= tokenizer.num_special_tokens_to_add(snake_case_       )
										A__					:    Tuple														= compute_effective_axis_dimension(
										    snake_case_      ,			fixed_dimension=OnnxConfig.default_fixed_sequence      ,			num_token_to_add=snake_case_       )

										# Generate dummy inputs according to compute batch and sequence
										A__					:    Any														= [""" """.join([tokenizer.unk_token]       ) * seq_length] * batch_size
										A__					:    List[str]														= dict(tokenizer(snake_case_      ,			return_tensors=snake_case_       )       )
										return common_inputs





			def 		lowerCamelCase				( self      ,			snake_case_      ,			snake_case_ = -1      ,			snake_case_ = -1      ,			snake_case_ = False      ,			snake_case_ = None      ,			):







										'''simple docstring'''


										if self.task in ["default", "seq2seq-lm"]:
																	A__					:    Dict														= self._generate_dummy_inputs_for_default_and_seqaseq_lm(
																	    snake_case_      ,			batch_size=snake_case_      ,			seq_length=snake_case_      ,			is_pair=snake_case_      ,			framework=snake_case_       )

										elif self.task == "causal-lm":
																	A__					:    Tuple														= self._generate_dummy_inputs_for_causal_lm(
																	    snake_case_      ,			batch_size=snake_case_      ,			seq_length=snake_case_      ,			is_pair=snake_case_      ,			framework=snake_case_       )
										else:
																	A__					:    Any														= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
																	    snake_case_      ,			batch_size=snake_case_      ,			seq_length=snake_case_      ,			is_pair=snake_case_      ,			framework=snake_case_       )

										return common_inputs





			def 		lowerCamelCase				( self      ,			snake_case_      ,			snake_case_      ,			snake_case_      ,			snake_case_       ):







										'''simple docstring'''


										if self.task in ["default", "seq2seq-lm"]:
																	A__					:    Union[str, Any]														= super()._flatten_past_key_values_(snake_case_      ,			snake_case_      ,			snake_case_      ,			snake_case_       )
										else:
																	A__					:    Optional[Any]														= super(snake_case_      ,			self       )._flatten_past_key_values_(
																	    snake_case_      ,			snake_case_      ,			snake_case_      ,			snake_case_       )






"""simple docstring"""



import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


_UpperCamelCase         =		get_logger(__name__)

def    _A(		lowerCAmelCase ,     lowerCAmelCase ,     lowerCAmelCase ,     lowerCAmelCase ,     lowerCAmelCase ,     lowerCAmelCase = None ,     ):
							# Model states
							for i, state in enumerate(lowerCAmelCase       ):
														A__					:    Tuple														= F'''{MODEL_NAME}.bin''' if i == 0 else F'''{MODEL_NAME}_{i}.bin'''
														A__					:    Union[str, Any]														= os.path.join(lowerCAmelCase ,     lowerCAmelCase       )
														save(lowerCAmelCase ,     lowerCAmelCase       )
														logger.info(F'''Model weights saved in {output_model_file}'''       )
							# Optimizer states
							for i, opt in enumerate(lowerCAmelCase       ):
														A__					:    Tuple														= opt.state_dict()
														A__					:    Union[str, Any]														= F'''{OPTIMIZER_NAME}.bin''' if i == 0 else F'''{OPTIMIZER_NAME}_{i}.bin'''
														A__					:    Tuple														= os.path.join(lowerCAmelCase ,     lowerCAmelCase       )
														save(lowerCAmelCase ,     lowerCAmelCase       )
														logger.info(F'''Optimizer state saved in {output_optimizer_file}'''       )
							# Scheduler states
							for i, scheduler in enumerate(lowerCAmelCase       ):
														A__					:    List[str]														= scheduler.state_dict()
														A__					:    Dict														= F'''{SCHEDULER_NAME}.bin''' if i == 0 else F'''{SCHEDULER_NAME}_{i}.bin'''
														A__					:    Optional[int]														= os.path.join(lowerCAmelCase ,     lowerCAmelCase       )
														save(lowerCAmelCase ,     lowerCAmelCase       )
														logger.info(F'''Scheduler state saved in {output_scheduler_file}'''       )
							# GradScaler state
							if scaler is not None:
														A__					:    Dict														= scaler.state_dict()
														A__					:    Optional[int]														= os.path.join(lowerCAmelCase ,     lowerCAmelCase       )
														torch.save(lowerCAmelCase ,     lowerCAmelCase       )
														logger.info(F'''Gradient scaler state saved in {output_scaler_file}'''       )
							# Random number generator states
							A__					:    Tuple														= {}
							A__					:    Any														= F'''{RNG_STATE_NAME}_{process_index}.pkl'''
							A__					:    Any														= random.getstate()
							A__					:    Any														= np.random.get_state()
							A__					:    int														= torch.get_rng_state()
							if is_xpu_available():
														A__					:    List[str]														= torch.xpu.get_rng_state_all()
							else:
														A__					:    Union[str, Any]														= torch.cuda.get_rng_state_all()
							if is_tpu_available():
														A__					:    str														= xm.get_rng_state()
							A__					:    str														= os.path.join(lowerCAmelCase ,     lowerCAmelCase       )
							torch.save(lowerCAmelCase ,     lowerCAmelCase       )
							logger.info(F'''Random states saved in {output_states_file}'''       )
							return output_dir

def    _A(		lowerCAmelCase ,     lowerCAmelCase ,     lowerCAmelCase ,     lowerCAmelCase ,     lowerCAmelCase ,     lowerCAmelCase=None ,     lowerCAmelCase=None ,     **lowerCAmelCase ,     ):
							if map_location not in [None, "cpu", "on_device"]:
														raise TypeError(
														    """Unsupported optimizer map location passed, please choose one of `None`, `'cpu'`, or `'on_device'`"""       )
							if map_location is None:
														A__					:    Union[str, Any]														= """cpu"""
							elif map_location == "on_device":
														A__					:    str														= PartialState().device
							# Model states
							for i, model in enumerate(lowerCAmelCase       ):
														A__					:    List[str]														= F'''{MODEL_NAME}.bin''' if i == 0 else F'''{MODEL_NAME}_{i}.bin'''
														A__					:    Optional[Any]														= os.path.join(lowerCAmelCase ,     lowerCAmelCase       )
														models[i].load_state_dict(torch.load(lowerCAmelCase ,     map_location=lowerCAmelCase       ) ,     **lowerCAmelCase       )
							logger.info("""All model weights loaded successfully"""       )

							# Optimizer states
							for i, opt in enumerate(lowerCAmelCase       ):
														A__					:    str														= F'''{OPTIMIZER_NAME}.bin''' if i == 0 else F'''{OPTIMIZER_NAME}_{i}.bin'''
														A__					:    Union[str, Any]														= os.path.join(lowerCAmelCase ,     lowerCAmelCase       )
														A__					:    List[str]														= torch.load(lowerCAmelCase ,     map_location=lowerCAmelCase       )
														optimizers[i].load_state_dict(lowerCAmelCase       )
							logger.info("""All optimizer states loaded successfully"""       )

							# Scheduler states
							for i, scheduler in enumerate(lowerCAmelCase       ):
														A__					:    Any														= F'''{SCHEDULER_NAME}.bin''' if i == 0 else F'''{SCHEDULER_NAME}_{i}.bin'''
														A__					:    Optional[int]														= os.path.join(lowerCAmelCase ,     lowerCAmelCase       )
														scheduler.load_state_dict(torch.load(lowerCAmelCase       )       )
							logger.info("""All scheduler states loaded successfully"""       )

							# GradScaler state
							if scaler is not None:
														A__					:    Optional[Any]														= os.path.join(lowerCAmelCase ,     lowerCAmelCase       )
														scaler.load_state_dict(torch.load(lowerCAmelCase       )       )
														logger.info("""GradScaler state loaded successfully"""       )

							# Random states
							try:
														A__					:    Union[str, Any]														= torch.load(os.path.join(lowerCAmelCase ,     F'''{RNG_STATE_NAME}_{process_index}.pkl'''       )       )
														random.setstate(states["""random_state"""]       )
														np.random.set_state(states["""numpy_random_seed"""]       )
														torch.set_rng_state(states["""torch_manual_seed"""]       )
														if is_xpu_available():
																					torch.xpu.set_rng_state_all(states["""torch_xpu_manual_seed"""]       )
														else:
																					torch.cuda.set_rng_state_all(states["""torch_cuda_manual_seed"""]       )
														if is_tpu_available():
																					xm.set_rng_state(states["""xm_seed"""]       )
														logger.info("""All random states loaded successfully"""       )
							except Exception:
														logger.info("""Could not load random states"""       )

def    _A(		lowerCAmelCase ,     lowerCAmelCase ,     lowerCAmelCase = 0       ):
							A__					:    int														= Path(lowerCAmelCase       ) / F'''custom_checkpoint_{index}.pkl'''
							logger.info(F'''Saving the state of {get_pretty_name(lowerCAmelCase       )} to {save_location}'''       )
							torch.save(obj.state_dict() ,     lowerCAmelCase       )



def    _A(		lowerCAmelCase ,     lowerCAmelCase ,     lowerCAmelCase = 0       ):
							A__					:    List[str]														= F'''{path}/custom_checkpoint_{index}.pkl'''
							logger.info(F'''Loading the state of {get_pretty_name(lowerCAmelCase       )} from {load_location}'''       )
							obj.load_state_dict(torch.load(lowerCAmelCase ,     map_location="""cpu"""       )       )





