





'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
	_a      :				str											=					None


_a      :				Optional[Any]											=					logging.get_logger(__name__)

_a      :				Dict											=					{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

_a      :				Dict											=					{
    """vocab_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json"""
        ),
    },
}

_a      :				Any											=					{
    """xlm-roberta-base""": 5_1_2,
    """xlm-roberta-large""": 5_1_2,
    """xlm-roberta-large-finetuned-conll02-dutch""": 5_1_2,
    """xlm-roberta-large-finetuned-conll02-spanish""": 5_1_2,
    """xlm-roberta-large-finetuned-conll03-english""": 5_1_2,
    """xlm-roberta-large-finetuned-conll03-german""": 5_1_2,
}





class 					_UpperCAmelCase							(		lowerCAmelCase_     ):
				a     :     Union[str, Any]     						=VOCAB_FILES_NAMES
				a     :     str     						=PRETRAINED_VOCAB_FILES_MAP
				a     :     Union[str, Any]     						=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				a     :     str     						=["input_ids", "attention_mask"]
				a     :     Tuple     						=XLMRobertaTokenizer
				def __init__(						self,__SCREAMING_SNAKE_CASE=None,__SCREAMING_SNAKE_CASE=None,__SCREAMING_SNAKE_CASE="<s>",__SCREAMING_SNAKE_CASE="</s>",__SCREAMING_SNAKE_CASE="</s>",__SCREAMING_SNAKE_CASE="<s>",__SCREAMING_SNAKE_CASE="<unk>",__SCREAMING_SNAKE_CASE="<pad>",__SCREAMING_SNAKE_CASE="<mask>",**__SCREAMING_SNAKE_CASE,):



							'''simple docstring'''

							__lowerCAmelCase      		=       AddedToken(UpperCamelCase_,lstrip=UpperCamelCase_,rstrip=UpperCamelCase_    ) if isinstance(UpperCamelCase_,UpperCamelCase_    ) else mask_token

							super().__init__(
							    UpperCamelCase_,tokenizer_file=UpperCamelCase_,bos_token=UpperCamelCase_,eos_token=UpperCamelCase_,sep_token=UpperCamelCase_,cls_token=UpperCamelCase_,unk_token=UpperCamelCase_,pad_token=UpperCamelCase_,mask_token=UpperCamelCase_,**UpperCamelCase_,)

							__lowerCAmelCase      		=       vocab_file
							__lowerCAmelCase      		=       False if not self.vocab_file else True
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE = None    ):



							'''simple docstring'''

							if token_ids_a is None:
										return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							__lowerCAmelCase      		=       [self.cls_token_id]
							__lowerCAmelCase      		=       [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE = None    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       [self.sep_token_id]
							__lowerCAmelCase      		=       [self.cls_token_id]

							if token_ids_a is None:
										return len(cls + token_ids_a + sep    ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]







				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE = None    ):



							'''simple docstring'''

							if not self.can_save_slow_tokenizer:
										raise ValueError(
										    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
										    """tokenizer."""    )

							if not os.path.isdir(UpperCamelCase_    ):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory.'    )
										return
							__lowerCAmelCase      		=       os.path.join(
							    UpperCamelCase_,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

							if os.path.abspath(self.vocab_file    ) != os.path.abspath(UpperCamelCase_    ):
										copyfile(self.vocab_file,UpperCamelCase_    )

							return (out_vocab_file,)








'''simple docstring'''







import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_a      :				Tuple											=					datasets.utils.logging.get_logger(__name__)





@dataclass
class 					_UpperCAmelCase							(		datasets.BuilderConfig     ):
		a     :     Optional[datasets.Features]     						=None
		a     :     str     						="utf-8"
		a     :     Optional[str]     						=None
		a     :     int     						=10 << 20  # 10MB
		a     :     bool     						=False
		a     :     str     						="line"





class 					_UpperCAmelCase							(		datasets.ArrowBasedBuilder     ):
		a     :     Optional[int]     						=TextConfig
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					return datasets.DatasetInfo(features=self.config.features    )
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					if not self.config.data_files:
								raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'    )
					__lowerCAmelCase      		=       dl_manager.download_and_extract(self.config.data_files    )
					if isinstance(__SCREAMING_SNAKE_CASE,(str, list, tuple)    ):
								__lowerCAmelCase      		=       data_files
								if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ):
											__lowerCAmelCase      		=       [files]
								__lowerCAmelCase      		=       [dl_manager.iter_files(__SCREAMING_SNAKE_CASE    ) for file in files]
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN,gen_kwargs={"""files""": files}    )]
					__lowerCAmelCase      		=       []
					for split_name, files in data_files.items():
								if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ):
											__lowerCAmelCase      		=       [files]
								__lowerCAmelCase      		=       [dl_manager.iter_files(__SCREAMING_SNAKE_CASE    ) for file in files]
								splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE,gen_kwargs={"""files""": files}    )    )
					return splits
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					if self.config.features is not None:
								__lowerCAmelCase      		=       self.config.features.arrow_schema
								if all(not require_storage_cast(__SCREAMING_SNAKE_CASE    ) for feature in self.config.features.values()    ):
											# cheaper cast
											__lowerCAmelCase      		=       pa_table.cast(__SCREAMING_SNAKE_CASE    )
								else:
											# more expensive cast; allows str <-> int/float or str to Audio for example
											__lowerCAmelCase      		=       table_cast(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
								return pa_table
					else:
								return pa_table.cast(pa.schema({"""text""": pa.string()}    )    )







		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       list(self.config.features    ) if self.config.features is not None else ["""text"""]
					for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE    )    ):
								# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
								with open(__SCREAMING_SNAKE_CASE,encoding=self.config.encoding,errors=self.config.errors    ) as f:
											if self.config.sample_by == "line":
														__lowerCAmelCase      		=       0
														while True:
																	__lowerCAmelCase      		=       f.read(self.config.chunksize    )
																	if not batch:
																				break
																	batch += f.readline()  # finish current line
																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																	__lowerCAmelCase      		=       StringIO(__SCREAMING_SNAKE_CASE    ).readlines()
																	if not self.config.keep_linebreaks:
																				__lowerCAmelCase      		=       [line.rstrip("""\n"""    ) for line in batch]
																	__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array(__SCREAMING_SNAKE_CASE    )],names=__SCREAMING_SNAKE_CASE    )
																	# Uncomment for debugging (will print the Arrow table size and elements)
																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
																	batch_idx += 1
											elif self.config.sample_by == "paragraph":
														__lowerCAmelCase      		=       0
														__lowerCAmelCase      		=       """"""
														while True:
																	__lowerCAmelCase      		=       f.read(self.config.chunksize    )
																	if not new_batch:
																				break
																	batch += new_batch
																	batch += f.readline()  # finish current line
																	__lowerCAmelCase      		=       batch.split("""\n\n"""    )
																	__lowerCAmelCase      		=       pa.Table.from_arrays(
																	    [pa.array([example for example in batch[:-1] if example]    )],names=__SCREAMING_SNAKE_CASE    )
																	# Uncomment for debugging (will print the Arrow table size and elements)
																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
																	batch_idx += 1
																	__lowerCAmelCase      		=       batch[-1]
														if batch:
																	__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array([batch]    )],names=__SCREAMING_SNAKE_CASE    )
																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
											elif self.config.sample_by == "document":
														__lowerCAmelCase      		=       f.read()
														__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array([text]    )],names=__SCREAMING_SNAKE_CASE    )
														yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE    )

