



'''simple docstring'''






import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


A_				:					Union[str, Any]							=      logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
A_				:					int							=      direct_transformers_import(Path(__file__).parent)


A_				:					List[str]							=      {
    """AutoTokenizer""": """PreTrainedTokenizerBase""",
    """AutoFeatureExtractor""": """FeatureExtractionMixin""",
    """AutoImageProcessor""": """ImageProcessingMixin""",
}



class 				lowercase			(    _lowerCamelCase							):


 """simple docstring"""


 UpperCAmelCase		      =    ["""feature_extractor""", """tokenizer"""]
 # Names need to be attr_class for attr in attributes
 UpperCAmelCase		      =    None
 UpperCAmelCase		      =    None
 UpperCAmelCase		      =    None







 def __init__(				self    ,*a_    ,**a_				)						->  List[Any]:
        # Sanitize args and kwargs
        for key in kwargs:
               if key not in self.attributes:
                      raise TypeError(f'''Unexpected keyword argument {key}.'''				)
        for arg, attribute_name in zip(a__    ,self.attributes				):
               if attribute_name in kwargs:
                      raise TypeError(f'''Got multiple values for argument {attribute_name}.'''				)
               else:
                      _UpperCAmelCase    :		Dict     	=					arg

        if len(a__				) != len(self.attributes				):
               raise ValueError(
                   f'''This processor requires {len(self.attributes				)} arguments: {', '.join(self.attributes				)}. Got '''
                   f'''{len(a__				)} arguments instead.'''				)

        # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
        for attribute_name, arg in kwargs.items():
               _UpperCAmelCase    :		List[Any]     	=					getattr(self    ,f'''{attribute_name}_class'''				)
               # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
               _UpperCAmelCase    :		List[str]     	=					AUTO_TO_BASE_CLASS_MAPPING.get(a__    ,a__				)
               if isinstance(a__    ,a__				):
                      _UpperCAmelCase    :		List[Any]     	=					tuple(getattr(a__    ,a__				) for n in class_name if n is not None				)
               else:
                      _UpperCAmelCase    :		List[Any]     	=					getattr(a__    ,a__				)

               if not isinstance(a__    ,a__				):
                      raise ValueError(
                          f'''Received a {type(a__				).__name__} for argument {attribute_name}, but a {class_name} was expected.'''				)

               setattr(self    ,a__    ,a__				)







 def __repr__(				self				)						->  Any:
        _UpperCAmelCase    :		Optional[int]     	=					[f'''- {name}: {repr(getattr(self    ,a__				)				)}''' for name in self.attributes]
        _UpperCAmelCase    :		List[str]     	=					"""\n""".join(a__				)
        return f'''{self.__class__.__name__}:\n{attributes_repr}'''







 def _snake_case					(				self    ,a_    ,a_ = False    ,**a_				)						->  Optional[Any]:
        os.makedirs(a__    ,exist_ok=a__				)

        if push_to_hub:
               _UpperCAmelCase    :		Optional[int]     	=					kwargs.pop("""commit_message"""    ,a__				)
               _UpperCAmelCase    :		Optional[Any]     	=					kwargs.pop("""repo_id"""    ,save_directory.split(os.path.sep				)[-1]				)
               _UpperCAmelCase    :		Optional[int]     	=					self._create_repo(a__    ,**a__				)
               _UpperCAmelCase    :		str     	=					self._get_files_timestamps(a__				)
        # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
        # loaded from the Hub.
        if self._auto_class is not None:
               _UpperCAmelCase    :		Optional[int]     	=					[getattr(self    ,a__				) for attribute_name in self.attributes]
               _UpperCAmelCase    :		Dict     	=					[(a.init_kwargs if isinstance(a__    ,a__				) else a) for a in attrs]
               custom_object_save(self    ,a__    ,config=a__				)

        for attribute_name in self.attributes:
               _UpperCAmelCase    :		int     	=					getattr(self    ,a__				)
               # Include the processor class in the attribute config so this processor can then be reloaded with the
               # `AutoProcessor` API.
               if hasattr(a__    ,"""_set_processor_class"""				):
                      attribute._set_processor_class(self.__class__.__name__				)
               attribute.save_pretrained(a__				)

        if self._auto_class is not None:
               # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
               for attribute_name in self.attributes:
                      _UpperCAmelCase    :		List[str]     	=					getattr(self    ,a__				)
                      if isinstance(a__    ,a__				):
                             del attribute.init_kwargs["auto_map"]

        if push_to_hub:
               self._upload_modified_files(
                   a__    ,a__    ,a__    ,commit_message=a__    ,token=kwargs.get("""use_auth_token"""				)    ,)







 @classmethod
 def _snake_case					(				cls    ,a_    ,a_ = None    ,a_ = False    ,a_ = False    ,a_ = None    ,a_ = "main"    ,**a_    ,)						->  List[Any]:
        _UpperCAmelCase    :		Optional[Any]     	=					cache_dir
        _UpperCAmelCase    :		Optional[Any]     	=					force_download
        _UpperCAmelCase    :		Dict     	=					local_files_only
        _UpperCAmelCase    :		Optional[Any]     	=					revision

        _UpperCAmelCase    :		str     	=					kwargs.pop("""use_auth_token"""    ,a__				)
        if use_auth_token is not None:
               warnings.warn(
                   """The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."""    ,a__				)
               if token is not None:
                      raise ValueError(
                          """`token` and `use_auth_token` are both specified. Please set only the argument `token`."""				)
               _UpperCAmelCase    :		str     	=					use_auth_token

        if token is not None:
               # change to `token` in a follow-up PR
               _UpperCAmelCase    :		Optional[int]     	=					token

        _UpperCAmelCase    :		Union[str, Any]     	=					cls._get_arguments_from_pretrained(a__    ,**a__				)
        return cls(*a__				)







 @classmethod
 def _snake_case					(				cls    ,a_="AutoProcessor"				)						->  List[str]:
        if not isinstance(a__    ,a__				):
               _UpperCAmelCase    :		Tuple     	=					auto_class.__name__

        import transformers.models.auto as auto_module

        if not hasattr(a__    ,a__				):
               raise ValueError(f'''{auto_class} is not a valid auto class.'''				)

        _UpperCAmelCase    :		str     	=					auto_class







 @classmethod
 def _snake_case					(				cls    ,a_    ,**a_				)						->  Optional[Any]:
        _UpperCAmelCase    :		Any     	=					[]
        for attribute_name in cls.attributes:
               _UpperCAmelCase    :		List[str]     	=					getattr(cls    ,f'''{attribute_name}_class'''				)
               if isinstance(a__    ,a__				):
                      _UpperCAmelCase    :		str     	=					tuple(getattr(a__    ,a__				) if n is not None else None for n in class_name				)
                      _UpperCAmelCase    :		List[str]     	=					kwargs.get("""use_fast"""    ,a__				)
                      if use_fast and classes[1] is not None:
                             _UpperCAmelCase    :		Optional[Any]     	=					classes[1]
                      else:
                             _UpperCAmelCase    :		List[str]     	=					classes[0]
               else:
                      _UpperCAmelCase    :		Union[str, Any]     	=					getattr(a__    ,a__				)

               args.append(attribute_class.from_pretrained(a__    ,**a__				)				)
        return args







 @property
 def _snake_case					(				self				)						->  Optional[int]:
        _UpperCAmelCase    :		List[Any]     	=					getattr(self    ,self.attributes[0]				)
        return getattr(a__    ,"""model_input_names"""    ,a__				)







A_				:					Tuple							=      copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
   A_				:					str							=      ProcessorMixin.push_to_hub.__doc__.format(
       object="""processor""", object_class="""AutoProcessor""", object_files="""processor files"""
   )










"""simple docstring"""






from typing import Callable, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_      =				logging.get_logger(__name__)

lowerCAmelCase_      =				{
    '''microsoft/prophetnet-large-uncased''': (
        '''https://huggingface.co/microsoft/prophetnet-large-uncased/resolve/main/config.json'''
    ),
}



class 						lowerCAmelCase       (   snake_case				):
       lowerCAmelCase__    			=	"""prophetnet"""
       lowerCAmelCase__    			=	["""past_key_values"""]
       lowerCAmelCase__    			=	{
           """num_attention_heads""": """num_encoder_attention_heads""",
       }






       def __init__( self     ,	a__ = 0.1     ,	a__ = "gelu"     ,	a__ = 3_05_22     ,	a__ = 10_24     ,	a__ = 40_96     ,	a__ = 12     ,	a__ = 16     ,	a__ = 40_96     ,	a__ = 12     ,	a__ = 16     ,	a__ = 0.1     ,	a__ = 0.1     ,	a__ = 5_12     ,	a__ = 0.02     ,	a__ = True     ,	a__ = True     ,	a__ = 0     ,	a__ = 2     ,	a__ = 32     ,	a__ = 1_28     ,	a__ = False     ,	a__ = 0.0     ,	a__ = True     ,	a__ = 0     ,	a__ = 1     ,	a__ = 2     ,	**a__     ,	):
              _UpperCAmelCase			     =  vocab_size
              _UpperCAmelCase			     =  hidden_size
              _UpperCAmelCase			     =  encoder_ffn_dim
              _UpperCAmelCase			     =  num_encoder_layers
              _UpperCAmelCase			     =  num_encoder_attention_heads
              _UpperCAmelCase			     =  decoder_ffn_dim
              _UpperCAmelCase			     =  num_decoder_layers
              _UpperCAmelCase			     =  num_decoder_attention_heads
              _UpperCAmelCase			     =  max_position_embeddings
              _UpperCAmelCase			     =  init_std  # Normal(0, this parameter)
              _UpperCAmelCase			     =  activation_function

              # parameters for prophetnet
              _UpperCAmelCase			     =  ngram
              _UpperCAmelCase			     =  num_buckets
              _UpperCAmelCase			     =  relative_max_distance
              _UpperCAmelCase			     =  disable_ngram_loss
              _UpperCAmelCase			     =  eps

              # 3 Types of Dropout
              _UpperCAmelCase			     =  attention_dropout
              _UpperCAmelCase			     =  activation_dropout
              _UpperCAmelCase			     =  dropout

              _UpperCAmelCase			     =  use_cache

              super().__init__(
                  pad_token_id=a__     ,	bos_token_id=a__     ,	eos_token_id=a__     ,	is_encoder_decoder=a__     ,	add_cross_attention=a__     ,	decoder_start_token_id=a__     ,	**a__     ,	)






       @property
       def 				__A			( self  ):
              return self.num_encoder_layers + self.num_decoder_layers






       @num_hidden_layers.setter
       def 				__A			( self     ,	a__  ):
              raise NotImplementedError(
                  'This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and'
                  ' `num_decoder_layers`.'  )






