




'''simple docstring'''





import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


a		:	Tuple       =      get_logger(__name__)


if TYPE_CHECKING:
  from .builder import DatasetBuilder





def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_ = None						):
    _a				:							Optional[int]							       =      importlib.import_module(UpperCAmelCase_						)

    # TODO(QL): always update the module to add subsequent new authentication
    if hasattr(UpperCAmelCase_   ,	"""_patched_for_streaming"""						) and module._patched_for_streaming:
        return

    def wrap_auth(UpperCAmelCase_						):
        @wraps(UpperCAmelCase_						)
        def wrapper(*UpperCAmelCase_   ,	**UpperCAmelCase_						):
            return function(*UpperCAmelCase_   ,	use_auth_token=UpperCAmelCase_   ,	**UpperCAmelCase_						)

        _a				:							List[str]							       =      """wrap_auth"""
        return wrapper

    # open files in a streaming fashion
    patch_submodule(UpperCAmelCase_   ,	"""open"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    patch_submodule(UpperCAmelCase_   ,	"""os.listdir"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    patch_submodule(UpperCAmelCase_   ,	"""os.walk"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    patch_submodule(UpperCAmelCase_   ,	"""glob.glob"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    # allow to navigate in remote zip files
    patch_submodule(UpperCAmelCase_   ,	"""os.path.join"""   ,	UpperCAmelCase_						).start()
    patch_submodule(UpperCAmelCase_   ,	"""os.path.dirname"""   ,	UpperCAmelCase_						).start()
    patch_submodule(UpperCAmelCase_   ,	"""os.path.basename"""   ,	UpperCAmelCase_						).start()
    patch_submodule(UpperCAmelCase_   ,	"""os.path.relpath"""   ,	UpperCAmelCase_						).start()
    patch_submodule(UpperCAmelCase_   ,	"""os.path.split"""   ,	UpperCAmelCase_						).start()
    patch_submodule(UpperCAmelCase_   ,	"""os.path.splitext"""   ,	UpperCAmelCase_						).start()
    # allow checks on paths
    patch_submodule(UpperCAmelCase_   ,	"""os.path.exists"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    patch_submodule(UpperCAmelCase_   ,	"""os.path.isdir"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    patch_submodule(UpperCAmelCase_   ,	"""os.path.isfile"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    patch_submodule(UpperCAmelCase_   ,	"""os.path.getsize"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    patch_submodule(UpperCAmelCase_   ,	"""pathlib.Path"""   ,	UpperCAmelCase_						).start()
    # file readers
    patch_submodule(UpperCAmelCase_   ,	"""gzip.open"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    patch_submodule(UpperCAmelCase_   ,	"""numpy.load"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    patch_submodule(UpperCAmelCase_   ,	"""pandas.read_csv"""   ,	wrap_auth(UpperCAmelCase_						)   ,	attrs=["""__version__"""]						).start()
    patch_submodule(UpperCAmelCase_   ,	"""pandas.read_excel"""   ,	wrap_auth(UpperCAmelCase_						)   ,	attrs=["""__version__"""]						).start()
    patch_submodule(UpperCAmelCase_   ,	"""scipy.io.loadmat"""   ,	wrap_auth(UpperCAmelCase_						)   ,	attrs=["""__version__"""]						).start()
    patch_submodule(UpperCAmelCase_   ,	"""xml.etree.ElementTree.parse"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    patch_submodule(UpperCAmelCase_   ,	"""xml.dom.minidom.parse"""   ,	wrap_auth(UpperCAmelCase_						)						).start()
    _a				:							Optional[Any]							       =      True





def UpperCAmelCase    (       UpperCAmelCase_						):
    extend_module_for_streaming(builder.__module__   ,	use_auth_token=builder.use_auth_token						)
    # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
    if not builder.__module__.startswith("""datasets."""						):  # check that it's not a packaged builder like csv
        for imports in get_imports(inspect.getfile(builder.__class__						)						):
            if imports[0] == "internal":
                _a				:							Optional[int]							       =      imports[1]
                _a				:							List[str]							       =      """.""".join(builder.__module__.split("""."""						)[:-1] + [internal_import_name]						)
                extend_module_for_streaming(UpperCAmelCase_   ,	use_auth_token=builder.use_auth_token						)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
    from .builder import DatasetBuilder

    _a				:							Dict							       =      [
        cls.__module__
        for cls in type(UpperCAmelCase_						).__mro__[1:]  # make sure it's not the same module we've already patched
        if issubclass(cls   ,	UpperCAmelCase_						) and cls.__module__ != DatasetBuilder.__module__
    ]  # check it's not a standard builder from datasets.builder
    for module in parent_builder_modules:
        extend_module_for_streaming(UpperCAmelCase_   ,	use_auth_token=builder.use_auth_token						)








'''simple docstring'''





import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_reformer import ReformerTokenizer
else:
  a		:	Optional[int]       =      None


a		:	Optional[Any]       =      logging.get_logger(__name__)


a		:	Union[str, Any]       =      '''▁'''

a		:	Any       =      {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

a		:	List[Any]       =      {
    '''vocab_file''': {
        '''google/reformer-crime-and-punishment''': (
            '''https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'''
        )
    },
    '''tokenizer_file''': {
        '''google/reformer-crime-and-punishment''': (
            '''https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json'''
        )
    },
}

a		:	Any       =      {
    '''google/reformer-crime-and-punishment''': 5_2_4_2_8_8,
}




class      UpperCamelCase       (	snake_case__       ):
  a     :str          =					VOCAB_FILES_NAMES
  a     :List[Any]          =					PRETRAINED_VOCAB_FILES_MAP
  a     :Optional[int]          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a     :Any          =					['input_ids', 'attention_mask']
  a     :int          =					ReformerTokenizer



  def __init__(			self		,     __lowerCAmelCase=None		,     __lowerCAmelCase=None		,     __lowerCAmelCase="</s>"		,     __lowerCAmelCase="<unk>"		,     __lowerCAmelCase=[]		,     **__lowerCAmelCase		,     ):







      '''simple docstring'''


      super().__init__(
          __lowerCAmelCase		,     tokenizer_file=__lowerCAmelCase		,     eos_token=__lowerCAmelCase		,     unk_token=__lowerCAmelCase		,     additional_special_tokens=__lowerCAmelCase		,     **__lowerCAmelCase		,     )

      _a				:							List[str]							       =      vocab_file
      _a				:							Optional[int]							       =      False if not self.vocab_file else True



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase = None ):







      '''simple docstring'''


      if not self.can_save_slow_tokenizer:
          raise ValueError(
              """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
              """tokenizer.""" )

      if not os.path.isdir(__lowerCAmelCase ):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
          return
      _a				:							List[str]							       =      os.path.join(
          __lowerCAmelCase		,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

      if os.path.abspath(self.vocab_file ) != os.path.abspath(__lowerCAmelCase ):
          copyfile(self.vocab_file		,     __lowerCAmelCase )

      return (out_vocab_file,)


