






'''simple docstring'''




import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
    import sqlitea

    import sqlalchemy


lowerCamelCase_							      =      datasets.utils.logging.get_logger(__name__)

@dataclass
class      _UpperCAmelCase    (  datasets.BuilderConfig		):




   """simple docstring"""

   snake_case					       =		None
   snake_case					       =		None
   snake_case					       =		None
   snake_case					       =		True
   snake_case					       =		None
   snake_case					       =		None
   snake_case					       =		None
   snake_case					       =		1_00_00
   snake_case					       =		None

   def 		lowerCAmelCase				(	self				:			Tuple	):





         '''simple docstring'''





         if self.sql is None:
               raise ValueError("sql must be specified"	)
         if self.con is None:
               raise ValueError("con must be specified"	)

   def 		lowerCAmelCase				(	self				:			str   ,   __UpperCAmelCase				:			dict   ,   __UpperCAmelCase				:			Optional[datasets.Features] = None   ,   ):





         '''simple docstring'''





         _A      =			config_kwargs.copy()
         # We need to stringify the Selectable object to make its hash deterministic

         # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
         _A      =			config_kwargs["sql"]
         if not isinstance(__UpperCAmelCase   ,   __UpperCAmelCase	):
               if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
                     import sqlalchemy

                     if isinstance(__UpperCAmelCase   ,   sqlalchemy.sql.Selectable	):
                           _A      =			sqlalchemy.create_engine(config_kwargs["con"].split("://"	)[0] + "://"	)
                           _A      =			str(sql.compile(dialect=engine.dialect	)	)
                           _A      =			sql_str
                     else:
                           raise TypeError(
                               f'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(__UpperCAmelCase	)}: {sql}'''	)
               else:
                     raise TypeError(
                         f'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(__UpperCAmelCase	)}: {sql}'''	)
         _A      =			config_kwargs["con"]
         if not isinstance(__UpperCAmelCase   ,   __UpperCAmelCase	):
               _A      =			id(__UpperCAmelCase	)
               logger.info(
                   f'''SQL connection \'con\' of type {type(__UpperCAmelCase	)} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'''	)

         return super().create_config_id(__UpperCAmelCase   ,   custom_features=__UpperCAmelCase	)

   @property
   def 		lowerCAmelCase				(	self				:			Any	):





         '''simple docstring'''





         _A      =			{
             "index_col": self.index_col,
             "columns": self.columns,
             "params": self.params,
             "coerce_float": self.coerce_float,
             "parse_dates": self.parse_dates,
         }
         return pd_read_sql_kwargs





class      _UpperCAmelCase    (  datasets.ArrowBasedBuilder		):




   """simple docstring"""

   snake_case					       =		SqlConfig

   def 		lowerCAmelCase				(	self				:			Optional[int]	):





         '''simple docstring'''





         return datasets.DatasetInfo(features=self.config.features	)

   def 		lowerCAmelCase				(	self				:			str   ,   __UpperCAmelCase				:			int	):





         '''simple docstring'''





         return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,   gen_kwargs={}	)]

   def 		lowerCAmelCase				(	self				:			Union[str, Any]   ,   __UpperCAmelCase				:			pa.Table	):





         '''simple docstring'''





         if self.config.features is not None:
               _A      =			self.config.features.arrow_schema
               if all(not require_storage_cast(__UpperCAmelCase	) for feature in self.config.features.values()	):
                     # cheaper cast
                     _A      =			pa.Table.from_arrays([pa_table[field.name] for field in schema]   ,   schema=__UpperCAmelCase	)
               else:
                     # more expensive cast; allows str <-> int/float or str to Audio for example
                     _A      =			table_cast(__UpperCAmelCase   ,   __UpperCAmelCase	)
         return pa_table

   def 		lowerCAmelCase				(	self				:			Dict	):





         '''simple docstring'''





         _A      =			self.config.chunksize
         _A      =			pd.read_sql(
             self.config.sql   ,   self.config.con   ,   chunksize=__UpperCAmelCase   ,   **self.config.pd_read_sql_kwargs	)
         _A      =			[sql_reader] if chunksize is None else sql_reader
         for chunk_idx, df in enumerate(__UpperCAmelCase	):
               _A      =			pa.Table.from_pandas(__UpperCAmelCase	)
               yield chunk_idx, self._cast_table(__UpperCAmelCase	)













'''simple docstring'''




import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_reformer import ReformerTokenizer
else:
    lowerCamelCase_							      =      None


lowerCamelCase_							      =      logging.get_logger(__name__)


lowerCamelCase_							      =      '''▁'''

lowerCamelCase_							      =      {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase_							      =      {
    '''vocab_file''': {
        '''google/reformer-crime-and-punishment''': (
            '''https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'''
        )
    },
    '''tokenizer_file''': {
        '''google/reformer-crime-and-punishment''': (
            '''https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json'''
        )
    },
}

lowerCamelCase_							      =      {
    '''google/reformer-crime-and-punishment''': 52_42_88,
}

class      _UpperCAmelCase    (  snake_case_		):




   """simple docstring"""

   snake_case					       =		VOCAB_FILES_NAMES
   snake_case					       =		PRETRAINED_VOCAB_FILES_MAP
   snake_case					       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   snake_case					       =		['''input_ids''', '''attention_mask''']
   snake_case					       =		ReformerTokenizer

   def __init__(	self				:			List[str]   ,   __UpperCAmelCase				:			List[str]=None   ,   __UpperCAmelCase				:			Any=None   ,   __UpperCAmelCase				:			List[Any]="</s>"   ,   __UpperCAmelCase				:			Optional[Any]="<unk>"   ,   __UpperCAmelCase				:			List[Any]=[]   ,   **__UpperCAmelCase				:			int   ,   ):





         '''simple docstring'''





         super().__init__(
             __UpperCAmelCase   ,   tokenizer_file=__UpperCAmelCase   ,   eos_token=__UpperCAmelCase   ,   unk_token=__UpperCAmelCase   ,   additional_special_tokens=__UpperCAmelCase   ,   **__UpperCAmelCase   ,   )

         _A      =			vocab_file
         _A      =			False if not self.vocab_file else True

   def 		lowerCAmelCase				(	self				:			Any   ,   __UpperCAmelCase				:			str   ,   __UpperCAmelCase				:			Optional[str] = None	):





         '''simple docstring'''





         if not self.can_save_slow_tokenizer:
               raise ValueError(
                   "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                   "tokenizer."	)

         if not os.path.isdir(__UpperCAmelCase	):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
               return
         _A      =			os.path.join(
             __UpperCAmelCase   ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)

         if os.path.abspath(self.vocab_file	) != os.path.abspath(__UpperCAmelCase	):
               copyfile(self.vocab_file   ,   __UpperCAmelCase	)

         return (out_vocab_file,)





