
'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


_lowercase				:						int												= logging.get_logger(__name__)

_lowercase				:						str												= {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_lowercase				:						Optional[int]												= {
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
    "tokenizer_file": {
        "allenai/longformer-base-4096": (
            "https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"
        ),
    },
}

_lowercase				:						Optional[int]												= {
    "allenai/longformer-base-4096": 4_0_9_6,
    "allenai/longformer-large-4096": 4_0_9_6,
    "allenai/longformer-large-4096-finetuned-triviaqa": 4_0_9_6,
    "allenai/longformer-base-4096-extra.pos.embd.only": 4_0_9_6,
    "allenai/longformer-large-4096-extra.pos.embd.only": 4_0_9_6,
}






class 				lowerCAmelCase__     (						lowerCamelCase_  ):
					lowerCAmelCase_       =  VOCAB_FILES_NAMES
					lowerCAmelCase_       =  PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_       =  ['''input_ids''', '''attention_mask''']
					lowerCAmelCase_       =  LongformerTokenizer






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE="replace"       ,							__SCREAMING_SNAKE_CASE="<s>"       ,							__SCREAMING_SNAKE_CASE="</s>"       ,							__SCREAMING_SNAKE_CASE="</s>"       ,							__SCREAMING_SNAKE_CASE="<s>"       ,							__SCREAMING_SNAKE_CASE="<unk>"       ,							__SCREAMING_SNAKE_CASE="<pad>"       ,							__SCREAMING_SNAKE_CASE="<mask>"       ,							__SCREAMING_SNAKE_CASE=False       ,							__SCREAMING_SNAKE_CASE=True       ,							**__SCREAMING_SNAKE_CASE       ,							):


											"""simple docstring"""





											super().__init__(
											    __SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							tokenizer_file=__SCREAMING_SNAKE_CASE       ,							errors=__SCREAMING_SNAKE_CASE       ,							bos_token=__SCREAMING_SNAKE_CASE       ,							eos_token=__SCREAMING_SNAKE_CASE       ,							sep_token=__SCREAMING_SNAKE_CASE       ,							cls_token=__SCREAMING_SNAKE_CASE       ,							unk_token=__SCREAMING_SNAKE_CASE       ,							pad_token=__SCREAMING_SNAKE_CASE       ,							mask_token=__SCREAMING_SNAKE_CASE       ,							add_prefix_space=__SCREAMING_SNAKE_CASE       ,							trim_offsets=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE       ,							)

											lowercase_			:  Dict									= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
											if pre_tok_state.get('''add_prefix_space'''       ,							__SCREAMING_SNAKE_CASE	) != add_prefix_space:
																	lowercase_			:  str									= getattr(__SCREAMING_SNAKE_CASE       ,							pre_tok_state.pop('''type'''	)	)
																	lowercase_			:  Optional[Any]									= add_prefix_space
																	lowercase_			:  str									= pre_tok_class(**__SCREAMING_SNAKE_CASE	)

											lowercase_			:  Optional[Any]									= add_prefix_space

											lowercase_			:  Dict									= '''post_processor'''
											lowercase_			:  Dict									= getattr(self.backend_tokenizer       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)
											if tokenizer_component_instance:
																	lowercase_			:  Tuple									= json.loads(tokenizer_component_instance.__getstate__()	)

																	# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																	if "sep" in state:
																							lowercase_			:  str									= tuple(state['''sep''']	)
																	if "cls" in state:
																							lowercase_			:  Dict									= tuple(state['''cls''']	)

																	lowercase_			:  Any									= False

																	if state.get('''add_prefix_space'''       ,							__SCREAMING_SNAKE_CASE	) != add_prefix_space:
																							lowercase_			:  Any									= add_prefix_space
																							lowercase_			:  str									= True

																	if state.get('''trim_offsets'''       ,							__SCREAMING_SNAKE_CASE	) != trim_offsets:
																							lowercase_			:  List[Any]									= trim_offsets
																							lowercase_			:  Union[str, Any]									= True

																	if changes_to_apply:
																							lowercase_			:  int									= getattr(__SCREAMING_SNAKE_CASE       ,							state.pop('''type'''	)	)
																							lowercase_			:  Tuple									= component_class(**__SCREAMING_SNAKE_CASE	)
																							setattr(self.backend_tokenizer       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)






					@property
					def 			_snake_case					(     self	):


											"""simple docstring"""





											if self._mask_token is None:
																	if self.verbose:
																							logger.error('''Using mask_token, but it is not set yet.'''	)
																	return None
											return str(self._mask_token	)






					@mask_token.setter
					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  Optional[Any]									= AddedToken(__SCREAMING_SNAKE_CASE       ,							lstrip=__SCREAMING_SNAKE_CASE       ,							rstrip=__SCREAMING_SNAKE_CASE	) if isinstance(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	) else value
											lowercase_			:  Optional[Any]									= value






					def 			_snake_case					(     self       ,							*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  List[Any]									= kwargs.get('''is_split_into_words'''       ,							__SCREAMING_SNAKE_CASE	)
											assert self.add_prefix_space or not is_split_into_words, (
											    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self       ,							*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  Optional[int]									= kwargs.get('''is_split_into_words'''       ,							__SCREAMING_SNAKE_CASE	)

											assert self.add_prefix_space or not is_split_into_words, (
											    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = None	):


											"""simple docstring"""





											lowercase_			:  str									= self._tokenizer.model.save(__SCREAMING_SNAKE_CASE       ,							name=__SCREAMING_SNAKE_CASE	)
											return tuple(__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE=None	):


											"""simple docstring"""





											lowercase_			:  int									= [self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																	return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = None	):


											"""simple docstring"""





											lowercase_			:  Dict									= [self.sep_token_id]
											lowercase_			:  int									= [self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep	) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]





'''simple docstring'''







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = False       ,							__SCREAMING_SNAKE_CASE = False       ,							__SCREAMING_SNAKE_CASE = None       ,							**__SCREAMING_SNAKE_CASE       ,							):


											"""simple docstring"""





											super().__init__(
											    __SCREAMING_SNAKE_CASE       ,							split=__SCREAMING_SNAKE_CASE       ,							features=__SCREAMING_SNAKE_CASE       ,							cache_dir=__SCREAMING_SNAKE_CASE       ,							keep_in_memory=__SCREAMING_SNAKE_CASE       ,							streaming=__SCREAMING_SNAKE_CASE       ,							num_proc=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE       ,							)
											lowercase_			:  int									= path_or_paths if isinstance(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	) else {self.split: path_or_paths}
											lowercase_			:  List[Any]									= Csv(
											    cache_dir=__SCREAMING_SNAKE_CASE       ,							data_files=__SCREAMING_SNAKE_CASE       ,							features=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE       ,							)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											if self.streaming:
																	lowercase_			:  List[str]									= self.builder.as_streaming_dataset(split=self.split	)
											# Build regular (map-style) dataset
											else:
																	lowercase_			:  Any									= None
																	lowercase_			:  Optional[Any]									= None
																	lowercase_			:  str									= None
																	lowercase_			:  Optional[int]									= None

																	self.builder.download_and_prepare(
																	    download_config=__SCREAMING_SNAKE_CASE       ,							download_mode=__SCREAMING_SNAKE_CASE       ,							verification_mode=__SCREAMING_SNAKE_CASE       ,							base_path=__SCREAMING_SNAKE_CASE       ,							num_proc=self.num_proc       ,							)
																	lowercase_			:  Tuple									= self.builder.as_dataset(
																	    split=self.split       ,							verification_mode=__SCREAMING_SNAKE_CASE       ,							in_memory=self.keep_in_memory	)
											return dataset







class 				lowerCAmelCase__     :






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = None       ,							**__SCREAMING_SNAKE_CASE       ,							):


											"""simple docstring"""





											if num_proc is not None and num_proc <= 0:
																	raise ValueError(F'''num_proc {num_proc} must be an integer > 0.'''	)

											lowercase_			:  List[Any]									= dataset
											lowercase_			:  Dict									= path_or_buf
											lowercase_			:  int									= batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
											lowercase_			:  int									= num_proc
											lowercase_			:  Optional[int]									= '''utf-8'''
											lowercase_			:  Optional[Any]									= to_csv_kwargs






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  List[str]									= self.to_csv_kwargs.pop('''path_or_buf'''       ,							__SCREAMING_SNAKE_CASE	)
											lowercase_			:  Optional[Any]									= self.to_csv_kwargs.pop('''header'''       ,							__SCREAMING_SNAKE_CASE	)
											lowercase_			:  Dict									= self.to_csv_kwargs.pop('''index'''       ,							__SCREAMING_SNAKE_CASE	)

											if isinstance(self.path_or_buf       ,							(str, bytes, os.PathLike)	):
																	with open(self.path_or_buf       ,							'''wb+'''	) as buffer:
																							lowercase_			:  Any									= self._write(file_obj=__SCREAMING_SNAKE_CASE       ,							header=__SCREAMING_SNAKE_CASE       ,							index=__SCREAMING_SNAKE_CASE       ,							**self.to_csv_kwargs	)
											else:
																	lowercase_			:  str									= self._write(file_obj=self.path_or_buf       ,							header=__SCREAMING_SNAKE_CASE       ,							index=__SCREAMING_SNAKE_CASE       ,							**self.to_csv_kwargs	)
											return written






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_		,				lowercase_		,				lowercase_		,				lowercase_			:  int									= args

											lowercase_			:  List[str]									= query_table(
											    table=self.dataset.data       ,							key=slice(__SCREAMING_SNAKE_CASE       ,							offset + self.batch_size	)       ,							indices=self.dataset._indices       ,							)
											lowercase_			:  Dict									= batch.to_pandas().to_csv(
											    path_or_buf=__SCREAMING_SNAKE_CASE       ,							header=header if (offset == 0) else False       ,							index=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)
											return csv_str.encode(self.encoding	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  Dict									= 0

											if self.num_proc is None or self.num_proc == 1:
																	for offset in logging.tqdm(
																	    range(0       ,							len(self.dataset	)       ,							self.batch_size	)       ,							unit='''ba'''       ,							disable=not logging.is_progress_bar_enabled()       ,							desc='''Creating CSV from Arrow format'''       ,							):
																							lowercase_			:  Any									= self._batch_csv((offset, header, index, to_csv_kwargs)	)
																							written += file_obj.write(__SCREAMING_SNAKE_CASE	)

											else:
																	lowercase_		,				lowercase_			:  Union[str, Any]									= len(self.dataset	), self.batch_size
																	with multiprocessing.Pool(self.num_proc	) as pool:
																							for csv_str in logging.tqdm(
																							    pool.imap(
																							        self._batch_csv       ,							[(offset, header, index, to_csv_kwargs) for offset in range(0       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)]       ,							)       ,							total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size       ,							unit='''ba'''       ,							disable=not logging.is_progress_bar_enabled()       ,							desc='''Creating CSV from Arrow format'''       ,							):
																													written += file_obj.write(__SCREAMING_SNAKE_CASE	)

											return written



