from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_A				:							Dict							= logging.get_logger(__name__)

_A				:							Tuple							= {
    """google/bigbird-pegasus-large-arxiv""": (
        """https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json"""
    ),
    """google/bigbird-pegasus-large-pubmed""": (
        """https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json"""
    ),
    """google/bigbird-pegasus-large-bigpatent""": (
        """https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json"""
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}



class 				__snake_case  (     __SCREAMING_SNAKE_CASE						):


    '''simple docstring'''







    lowerCamelCase__						:		Dict							=  """bigbird_pegasus"""
    lowerCamelCase__						:		str							=  ["""past_key_values"""]
    lowerCamelCase__						:		Union[str, Any]							=  {
        """num_attention_heads""": """encoder_attention_heads""",
        """hidden_size""": """d_model""",
        """attention_probs_dropout_prob""": """attention_dropout""",
    }



    def __init__(					self      ,							A_=9_61_03      ,							A_=40_96      ,							A_=16      ,							A_=40_96      ,							A_=16      ,							A_=16      ,							A_=40_96      ,							A_=16      ,							A_=0.0      ,							A_=0.0      ,							A_=True      ,							A_=True      ,							A_="gelu_new"      ,							A_=10_24      ,							A_=0.1      ,							A_=0.0      ,							A_=0.0      ,							A_=0.02      ,							A_=2      ,							A_=0.0      ,							A_=True      ,							A_=0      ,							A_=2      ,							A_=1      ,							A_="block_sparse"      ,							A_=64      ,							A_=3      ,							A_=False      ,							**A_      ,							):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= vocab_size
        SCREAMING_SNAKE_CASE__       							= max_position_embeddings
        SCREAMING_SNAKE_CASE__       							= d_model
        SCREAMING_SNAKE_CASE__       							= encoder_ffn_dim
        SCREAMING_SNAKE_CASE__       							= encoder_layers
        SCREAMING_SNAKE_CASE__       							= encoder_attention_heads
        SCREAMING_SNAKE_CASE__       							= decoder_ffn_dim
        SCREAMING_SNAKE_CASE__       							= decoder_layers
        SCREAMING_SNAKE_CASE__       							= decoder_attention_heads
        SCREAMING_SNAKE_CASE__       							= dropout
        SCREAMING_SNAKE_CASE__       							= attention_dropout
        SCREAMING_SNAKE_CASE__       							= activation_dropout
        SCREAMING_SNAKE_CASE__       							= activation_function
        SCREAMING_SNAKE_CASE__       							= init_std
        SCREAMING_SNAKE_CASE__       							= encoder_layerdrop
        SCREAMING_SNAKE_CASE__       							= decoder_layerdrop
        SCREAMING_SNAKE_CASE__       							= classifier_dropout
        SCREAMING_SNAKE_CASE__       							= use_cache
        SCREAMING_SNAKE_CASE__       							= encoder_layers
        SCREAMING_SNAKE_CASE__       							= scale_embedding  # scale factor will be sqrt(d_model) if True

        # extra config
        SCREAMING_SNAKE_CASE__       							= attention_type
        SCREAMING_SNAKE_CASE__       							= block_size
        SCREAMING_SNAKE_CASE__       							= num_random_blocks
        SCREAMING_SNAKE_CASE__       							= use_bias

        super().__init__(
            pad_token_id=A_      ,							bos_token_id=A_      ,							eos_token_id=A_      ,							is_encoder_decoder=A_      ,							decoder_start_token_id=A_      ,							**A_      ,							)





class 				__snake_case  (     __SCREAMING_SNAKE_CASE						):


    '''simple docstring'''







    @property
    def    lowercase_				(					self    ):





        '''simple docstring'''

        if self.task in ["default", "seq2seq-lm"]:
            SCREAMING_SNAKE_CASE__       							= OrderedDict(
                [
                    ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                    ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                ]    )

            if self.use_past:
                SCREAMING_SNAKE_CASE__       							= {0: '''batch'''}
                SCREAMING_SNAKE_CASE__       							= {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
            else:
                SCREAMING_SNAKE_CASE__       							= {0: '''batch''', 1: '''decoder_sequence'''}
                SCREAMING_SNAKE_CASE__       							= {0: '''batch''', 1: '''decoder_sequence'''}

            if self.use_past:
                self.fill_with_past_key_values_(A_      ,							direction='''inputs'''    )
        elif self.task == "causal-lm":
            # TODO: figure this case out.
            SCREAMING_SNAKE_CASE__       							= OrderedDict(
                [
                    ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                    ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                ]    )
            if self.use_past:
                SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__       							= self.num_layers
                for i in range(A_    ):
                    SCREAMING_SNAKE_CASE__       							= {0: '''batch''', 2: '''past_sequence + sequence'''}
                    SCREAMING_SNAKE_CASE__       							= {0: '''batch''', 2: '''past_sequence + sequence'''}
        else:
            SCREAMING_SNAKE_CASE__       							= OrderedDict(
                [
                    ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                    ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                    ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
                    ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
                ]    )

        return common_inputs



    @property
    def    lowercase_				(					self    ):





        '''simple docstring'''

        if self.task in ["default", "seq2seq-lm"]:
            SCREAMING_SNAKE_CASE__       							= super().outputs
        else:
            SCREAMING_SNAKE_CASE__       							= super(A_      ,							self    ).outputs
            if self.use_past:
                SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__       							= self.num_layers
                for i in range(A_    ):
                    SCREAMING_SNAKE_CASE__       							= {0: '''batch''', 2: '''past_sequence + sequence'''}
                    SCREAMING_SNAKE_CASE__       							= {0: '''batch''', 2: '''past_sequence + sequence'''}
        return common_outputs



    def    lowercase_				(					self      ,							A_      ,							A_ = -1      ,							A_ = -1      ,							A_ = False      ,							A_ = None      ,							):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            A_      ,							A_      ,							A_      ,							A_      ,							A_    )

        # Generate decoder inputs
        SCREAMING_SNAKE_CASE__       							= seq_length if not self.use_past else 1
        SCREAMING_SNAKE_CASE__       							= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            A_      ,							A_      ,							A_      ,							A_      ,							A_    )
        SCREAMING_SNAKE_CASE__       							= {f'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
        SCREAMING_SNAKE_CASE__       							= dict(**A_      ,							**A_    )

        if self.use_past:
            if not is_torch_available():
                raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''    )
            else:
                import torch
            SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__       							= common_inputs['''input_ids'''].shape
            SCREAMING_SNAKE_CASE__       							= common_inputs['''decoder_input_ids'''].shape[1]
            SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__       							= self.num_attention_heads
            SCREAMING_SNAKE_CASE__       							= (
                batch,
                num_encoder_attention_heads,
                encoder_seq_length,
                self._config.hidden_size // num_encoder_attention_heads,
            )
            SCREAMING_SNAKE_CASE__       							= decoder_seq_length + 3
            SCREAMING_SNAKE_CASE__       							= (
                batch,
                num_decoder_attention_heads,
                decoder_past_length,
                self._config.hidden_size // num_decoder_attention_heads,
            )

            SCREAMING_SNAKE_CASE__       							= torch.cat(
                [common_inputs['''decoder_attention_mask'''], torch.ones(A_      ,							A_    )]      ,							dim=1    )

            SCREAMING_SNAKE_CASE__       							= []
            # If the number of encoder and decoder layers are present in the model configuration, both are considered
            SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__       							= self.num_layers
            SCREAMING_SNAKE_CASE__       							= min(A_      ,							A_    )
            SCREAMING_SNAKE_CASE__       							= max(A_      ,							A_    ) - min_num_layers
            SCREAMING_SNAKE_CASE__       							= '''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

            for _ in range(A_    ):
                common_inputs["past_key_values"].append(
                    (
                        torch.zeros(A_    ),
                        torch.zeros(A_    ),
                        torch.zeros(A_    ),
                        torch.zeros(A_    ),
                    )    )
            # TODO: test this.
            SCREAMING_SNAKE_CASE__       							= encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
            for _ in range(A_      ,							A_    ):
                common_inputs["past_key_values"].append((torch.zeros(A_    ), torch.zeros(A_    ))    )
        return common_inputs



    def    lowercase_				(					self      ,							A_      ,							A_ = -1      ,							A_ = -1      ,							A_ = False      ,							A_ = None      ,							):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            A_      ,							A_      ,							A_      ,							A_      ,							A_    )

        if self.use_past:
            if not is_torch_available():
                raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''    )
            else:
                import torch
            SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__       							= common_inputs['''input_ids'''].shape
            # Not using the same length for past_key_values
            SCREAMING_SNAKE_CASE__       							= seqlen + 2
            SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__       							= self.num_layers
            SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__       							= self.num_attention_heads
            SCREAMING_SNAKE_CASE__       							= (
                batch,
                num_encoder_attention_heads,
                past_key_values_length,
                self._config.hidden_size // num_encoder_attention_heads,
            )

            SCREAMING_SNAKE_CASE__       							= common_inputs['''attention_mask'''].dtype
            SCREAMING_SNAKE_CASE__       							= torch.cat(
                [common_inputs['''attention_mask'''], torch.ones(A_      ,							A_      ,							dtype=A_    )]      ,							dim=1    )
            SCREAMING_SNAKE_CASE__       							= [
                (torch.zeros(A_    ), torch.zeros(A_    )) for _ in range(A_    )
            ]
        return common_inputs



    def    lowercase_				(					self      ,							A_      ,							A_ = -1      ,							A_ = -1      ,							A_ = False      ,							A_ = None      ,							):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= compute_effective_axis_dimension(
            A_      ,							fixed_dimension=OnnxConfig.default_fixed_batch      ,							num_token_to_add=0    )

        # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
        SCREAMING_SNAKE_CASE__       							= tokenizer.num_special_tokens_to_add(A_    )
        SCREAMING_SNAKE_CASE__       							= compute_effective_axis_dimension(
            A_      ,							fixed_dimension=OnnxConfig.default_fixed_sequence      ,							num_token_to_add=A_    )

        # Generate dummy inputs according to compute batch and sequence
        SCREAMING_SNAKE_CASE__       							= [''' '''.join([tokenizer.unk_token]    ) * seq_length] * batch_size
        SCREAMING_SNAKE_CASE__       							= dict(tokenizer(A_      ,							return_tensors=A_    )    )
        return common_inputs



    def    lowercase_				(					self      ,							A_      ,							A_ = -1      ,							A_ = -1      ,							A_ = False      ,							A_ = None      ,							):





        '''simple docstring'''

        if self.task in ["default", "seq2seq-lm"]:
            SCREAMING_SNAKE_CASE__       							= self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                A_      ,							batch_size=A_      ,							seq_length=A_      ,							is_pair=A_      ,							framework=A_    )

        elif self.task == "causal-lm":
            SCREAMING_SNAKE_CASE__       							= self._generate_dummy_inputs_for_causal_lm(
                A_      ,							batch_size=A_      ,							seq_length=A_      ,							is_pair=A_      ,							framework=A_    )
        else:
            SCREAMING_SNAKE_CASE__       							= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                A_      ,							batch_size=A_      ,							seq_length=A_      ,							is_pair=A_      ,							framework=A_    )

        return common_inputs




    def    lowercase_				(					self      ,							A_      ,							A_      ,							A_      ,							A_    ):





        '''simple docstring'''

        if self.task in ["default", "seq2seq-lm"]:
            SCREAMING_SNAKE_CASE__       							= super()._flatten_past_key_values_(A_      ,							A_      ,							A_      ,							A_    )
        else:
            SCREAMING_SNAKE_CASE__       							= super(A_      ,							self    )._flatten_past_key_values_(
                A_      ,							A_      ,							A_      ,							A_    )







import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A				:							Any							= logging.get_logger(__name__)

_A				:							Any							= {
    """facebook/hubert-base-ls960""": """https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json""",
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}



class 				__snake_case  (     __SCREAMING_SNAKE_CASE						):


    '''simple docstring'''







    lowerCamelCase__						:		int							=  """hubert"""



    def __init__(					self      ,							A_=32      ,							A_=7_68      ,							A_=12      ,							A_=12      ,							A_=30_72      ,							A_="gelu"      ,							A_=0.1      ,							A_=0.1      ,							A_=0.1      ,							A_=True      ,							A_=0.0      ,							A_=0.1      ,							A_=0.1      ,							A_=0.02      ,							A_=1E-5      ,							A_="group"      ,							A_="gelu"      ,							A_=(5_12, 5_12, 5_12, 5_12, 5_12, 5_12, 5_12)      ,							A_=(5, 2, 2, 2, 2, 2, 2)      ,							A_=(10, 3, 3, 3, 3, 2, 2)      ,							A_=False      ,							A_=1_28      ,							A_=16      ,							A_=False      ,							A_=True      ,							A_=0.05      ,							A_=10      ,							A_=2      ,							A_=0.0      ,							A_=10      ,							A_=0      ,							A_="sum"      ,							A_=False      ,							A_=False      ,							A_=2_56      ,							A_=0      ,							A_=1      ,							A_=2      ,							**A_      ,							):





        '''simple docstring'''

        super().__init__(**A_      ,							pad_token_id=A_      ,							bos_token_id=A_      ,							eos_token_id=A_    )
        SCREAMING_SNAKE_CASE__       							= hidden_size
        SCREAMING_SNAKE_CASE__       							= feat_extract_norm
        SCREAMING_SNAKE_CASE__       							= feat_extract_activation
        SCREAMING_SNAKE_CASE__       							= list(A_    )
        SCREAMING_SNAKE_CASE__       							= list(A_    )
        SCREAMING_SNAKE_CASE__       							= list(A_    )
        SCREAMING_SNAKE_CASE__       							= conv_bias
        SCREAMING_SNAKE_CASE__       							= num_conv_pos_embeddings
        SCREAMING_SNAKE_CASE__       							= num_conv_pos_embedding_groups
        SCREAMING_SNAKE_CASE__       							= len(self.conv_dim    )
        SCREAMING_SNAKE_CASE__       							= num_hidden_layers
        SCREAMING_SNAKE_CASE__       							= intermediate_size
        SCREAMING_SNAKE_CASE__       							= hidden_act
        SCREAMING_SNAKE_CASE__       							= num_attention_heads
        SCREAMING_SNAKE_CASE__       							= hidden_dropout
        SCREAMING_SNAKE_CASE__       							= attention_dropout
        SCREAMING_SNAKE_CASE__       							= activation_dropout
        SCREAMING_SNAKE_CASE__       							= feat_proj_layer_norm
        SCREAMING_SNAKE_CASE__       							= feat_proj_dropout
        SCREAMING_SNAKE_CASE__       							= final_dropout
        SCREAMING_SNAKE_CASE__       							= layerdrop
        SCREAMING_SNAKE_CASE__       							= layer_norm_eps
        SCREAMING_SNAKE_CASE__       							= initializer_range
        SCREAMING_SNAKE_CASE__       							= vocab_size
        SCREAMING_SNAKE_CASE__       							= do_stable_layer_norm
        SCREAMING_SNAKE_CASE__       							= use_weighted_layer_sum
        SCREAMING_SNAKE_CASE__       							= classifier_proj_size

        if (
            (len(self.conv_stride    ) != self.num_feat_extract_layers)
            or (len(self.conv_kernel    ) != self.num_feat_extract_layers)
            or (len(self.conv_dim    ) != self.num_feat_extract_layers)
        ):
            raise ValueError(
                '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                f''' {len(self.conv_dim    )}`, `len(config.conv_stride) = {len(self.conv_stride    )}`,'''
                f''' `len(config.conv_kernel) = {len(self.conv_kernel    )}`.'''    )

        # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
        SCREAMING_SNAKE_CASE__       							= apply_spec_augment
        SCREAMING_SNAKE_CASE__       							= mask_time_prob
        SCREAMING_SNAKE_CASE__       							= mask_time_length
        SCREAMING_SNAKE_CASE__       							= mask_time_min_masks
        SCREAMING_SNAKE_CASE__       							= mask_feature_prob
        SCREAMING_SNAKE_CASE__       							= mask_feature_length
        SCREAMING_SNAKE_CASE__       							= mask_feature_min_masks

        # ctc loss
        SCREAMING_SNAKE_CASE__       							= ctc_loss_reduction
        SCREAMING_SNAKE_CASE__       							= ctc_zero_infinity




    @property
    def    lowercase_				(					self    ):





        '''simple docstring'''

        return functools.reduce(operator.mul      ,							self.conv_stride      ,							1    )
