



"""simple docstring"""






import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase					:					Optional[Any] 		=		logging.get_logger(__name__)

_lowercase					:					int 		=		{}

class 			__SCREAMING_SNAKE_CASE      ( lowerCAmelCase_  ):


					'''simple docstring'''





					_a       =				'fsmt_decoder'



					def __init__(			self				:    Any,    lowerCamelCase				:    Union[str, Any]=0,    lowerCamelCase				:    List[str]=0	)->   Tuple:
								super().__init__()
								lowerCamelCase__						:						List[str]						      =vocab_size
								lowerCamelCase__						:						int						      =bos_token_id





class 			__SCREAMING_SNAKE_CASE      ( lowerCAmelCase_  ):


					'''simple docstring'''





					_a       =				'fsmt'
					_a       =				{'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}



					def __init__(			self				:    int,    lowerCamelCase				:    Optional[Any]=["en", "de"],    lowerCamelCase				:    Any=4_2024,    lowerCamelCase				:    Tuple=4_2024,    lowerCamelCase				:    Dict="relu",    lowerCamelCase				:    str=1024,    lowerCamelCase				:    int=200,    lowerCamelCase				:    Union[str, Any]=1024,    lowerCamelCase				:    Dict=4096,    lowerCamelCase				:    Optional[int]=12,    lowerCamelCase				:    List[Any]=16,    lowerCamelCase				:    Optional[Any]=0.0,    lowerCamelCase				:    List[Any]=4096,    lowerCamelCase				:    int=12,    lowerCamelCase				:    Tuple=16,    lowerCamelCase				:    List[Any]=0.0,    lowerCamelCase				:    List[str]=0.0,    lowerCamelCase				:    int=0.1,    lowerCamelCase				:    Optional[int]=0.0,    lowerCamelCase				:    Union[str, Any]=0.02,    lowerCamelCase				:    List[Any]=2,    lowerCamelCase				:    List[Any]=True,    lowerCamelCase				:    str=True,    lowerCamelCase				:    int=False,    lowerCamelCase				:    Optional[Any]=5,    lowerCamelCase				:    Optional[Any]=1.0,    lowerCamelCase				:    Any=False,    lowerCamelCase				:    Union[str, Any]=True,    lowerCamelCase				:    Any=1,    lowerCamelCase				:    Optional[Any]=0,    lowerCamelCase				:    Tuple=2,    lowerCamelCase				:    Tuple=2,    **lowerCamelCase				:    Tuple,    )->   Dict:
								lowerCamelCase__						:						Tuple						      =langs
								lowerCamelCase__						:						str						      =src_vocab_size
								lowerCamelCase__						:						str						      =tgt_vocab_size
								lowerCamelCase__						:						List[Any]						      =d_model  # encoder_embed_dim and decoder_embed_dim

								lowerCamelCase__						:						Union[str, Any]						      =encoder_ffn_dim
								lowerCamelCase__						:						List[str]						      =encoder_layers
								lowerCamelCase__						:						Any						      =encoder_attention_heads
								lowerCamelCase__						:						Optional[int]						      =encoder_layerdrop
								lowerCamelCase__						:						Optional[Any]						      =decoder_layerdrop
								lowerCamelCase__						:						str						      =decoder_ffn_dim
								lowerCamelCase__						:						Union[str, Any]						      =decoder_layers
								lowerCamelCase__						:						Union[str, Any]						      =decoder_attention_heads
								lowerCamelCase__						:						Optional[Any]						      =max_position_embeddings
								lowerCamelCase__						:						Dict						      =init_std  # Normal(0, this parameter)
								lowerCamelCase__						:						List[str]						      =activation_function

								lowerCamelCase__						:						Optional[Any]						      =DecoderConfig(vocab_size=lowerCamelCase,    bos_token_id=lowerCamelCase	)
								if "decoder" in common_kwargs:
											del common_kwargs["decoder"]

								lowerCamelCase__						:						int						      =scale_embedding  # scale factor will be sqrt(d_model) if True

								# 3 Types of Dropout
								lowerCamelCase__						:						int						      =attention_dropout
								lowerCamelCase__						:						int						      =activation_dropout
								lowerCamelCase__						:						Optional[int]						      =dropout

								lowerCamelCase__						:						int						      =use_cache
								super().__init__(
								    pad_token_id=lowerCamelCase,    bos_token_id=lowerCamelCase,    eos_token_id=lowerCamelCase,    decoder_start_token_id=lowerCamelCase,    is_encoder_decoder=lowerCamelCase,    tie_word_embeddings=lowerCamelCase,    forced_eos_token_id=lowerCamelCase,    max_length=lowerCamelCase,    num_beams=lowerCamelCase,    length_penalty=lowerCamelCase,    early_stopping=lowerCamelCase,    **lowerCamelCase,    )




					def 						snake_case			(			self				:    List[str]	)->   int:
								lowerCamelCase__						:						List[str]						      =copy.deepcopy(self.__dict__	)
								lowerCamelCase__						:						Union[str, Any]						      =self.decoder.to_dict()
								lowerCamelCase__						:						int						      =self.__class__.model_type
								return output












"""simple docstring"""



import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


A_     =				get_logger(__name__)


if TYPE_CHECKING:
						from .builder import DatasetBuilder




def   _lowerCAmelCase		(				UpperCAmelCase__							:       Optional[int],    UpperCAmelCase__							:       Optional[Union[str, bool]] = None   )		->Optional[Any]:
						A__    :	int			=	importlib.import_module(UpperCAmelCase__   )

						# TODO(QL): always update the module to add subsequent new authentication
						if hasattr(UpperCAmelCase__,    """_patched_for_streaming"""   ) and module._patched_for_streaming:
												return

						def wrap_auth(UpperCAmelCase__							:       List[str]   ):
												@wraps(UpperCAmelCase__   )
												def wrapper(*UpperCAmelCase__							:       Dict,    **UpperCAmelCase__							:       Dict   ):
																		return function(*UpperCAmelCase__,    use_auth_token=UpperCAmelCase__,    **UpperCAmelCase__   )

												A__    :	List[Any]			=	"""wrap_auth"""
												return wrapper

						# open files in a streaming fashion
						patch_submodule(UpperCAmelCase__,    """open""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """os.listdir""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """os.walk""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """glob.glob""",    wrap_auth(UpperCAmelCase__   )   ).start()
						# allow to navigate in remote zip files
						patch_submodule(UpperCAmelCase__,    """os.path.join""",    UpperCAmelCase__   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.dirname""",    UpperCAmelCase__   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.basename""",    UpperCAmelCase__   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.relpath""",    UpperCAmelCase__   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.split""",    UpperCAmelCase__   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.splitext""",    UpperCAmelCase__   ).start()
						# allow checks on paths
						patch_submodule(UpperCAmelCase__,    """os.path.exists""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.isdir""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.isfile""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """os.path.getsize""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """pathlib.Path""",    UpperCAmelCase__   ).start()
						# file readers
						patch_submodule(UpperCAmelCase__,    """gzip.open""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """numpy.load""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """pandas.read_csv""",    wrap_auth(UpperCAmelCase__   ),    attrs=["""__version__"""]   ).start()
						patch_submodule(UpperCAmelCase__,    """pandas.read_excel""",    wrap_auth(UpperCAmelCase__   ),    attrs=["""__version__"""]   ).start()
						patch_submodule(UpperCAmelCase__,    """scipy.io.loadmat""",    wrap_auth(UpperCAmelCase__   ),    attrs=["""__version__"""]   ).start()
						patch_submodule(UpperCAmelCase__,    """xml.etree.ElementTree.parse""",    wrap_auth(UpperCAmelCase__   )   ).start()
						patch_submodule(UpperCAmelCase__,    """xml.dom.minidom.parse""",    wrap_auth(UpperCAmelCase__   )   ).start()
						A__    :	Any			=	True




def   _lowerCAmelCase		(				UpperCAmelCase__							:       "DatasetBuilder"   )		->Tuple:
						extend_module_for_streaming(builder.__module__,    use_auth_token=builder.use_auth_token   )
						# if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
						if not builder.__module__.startswith("""datasets."""   ):  # check that it's not a packaged builder like csv
												for imports in get_imports(inspect.getfile(builder.__class__   )   ):
																		if imports[0] == "internal":
																								A__    :	Any			=	imports[1]
																								A__    :	Any			=	""".""".join(builder.__module__.split("""."""   )[:-1] + [internal_import_name]   )
																								extend_module_for_streaming(UpperCAmelCase__,    use_auth_token=builder.use_auth_token   )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
						from .builder import DatasetBuilder

						A__    :	Any			=	[
						    cls.__module__
						    for cls in type(UpperCAmelCase__   ).__mro__[1:]  # make sure it's not the same module we've already patched
						    if issubclass(cls,    UpperCAmelCase__   ) and cls.__module__ != DatasetBuilder.__module__
						]  # check it's not a standard builder from datasets.builder
						for module in parent_builder_modules:
												extend_module_for_streaming(UpperCAmelCase__,    use_auth_token=builder.use_auth_token   )






