


'''simple docstring'''



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


__lowercase			:		Dict =			datasets.utils.logging.get_logger(__name__)





@dataclass
class 							__UpperCamelCase       (	datasets.BuilderConfig      ):
	A_			    =					None
	A_			    =					"utf-8"
	A_			    =					None
	A_			    =					10 << 20  # 10MB
	A_			    =					False
	A_			    =					"line"





class 							__UpperCamelCase       (	datasets.ArrowBasedBuilder      ):
	A_			    =					TextConfig




	def    __UpperCAmelCase							(     self							):

						'''simple docstring'''





						return datasets.DatasetInfo(features=self.config.features							)




	def    __UpperCAmelCase							(     self		,       __a							):

						'''simple docstring'''





						if not self.config.data_files:
											raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""							)
						__a    : Union[str, Any]      					=						dl_manager.download_and_extract(self.config.data_files							)
						if isinstance(__a		,       (str, list, tuple)							):
											__a    : Optional[int]      					=						data_files
											if isinstance(__a		,       __a							):
																__a    : int      					=						[files]
											__a    : Any      					=						[dl_manager.iter_files(__a							) for file in files]
											return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,       gen_kwargs={'files': files}							)]
						__a    : List[Any]      					=						[]
						for split_name, files in data_files.items():
											if isinstance(__a		,       __a							):
																__a    : Dict      					=						[files]
											__a    : List[str]      					=						[dl_manager.iter_files(__a							) for file in files]
											splits.append(datasets.SplitGenerator(name=__a		,       gen_kwargs={'files': files}							)							)
						return splits




	def    __UpperCAmelCase							(     self		,       __a							):

						'''simple docstring'''





						if self.config.features is not None:
											__a    : Any      					=						self.config.features.arrow_schema
											if all(not require_storage_cast(__a							) for feature in self.config.features.values()							):
																# cheaper cast
																__a    : Tuple      					=						pa_table.cast(__a							)
											else:
																# more expensive cast; allows str <-> int/float or str to Audio for example
																__a    : Any      					=						table_cast(__a		,       __a							)
											return pa_table
						else:
											return pa_table.cast(pa.schema({'text': pa.string()}							)							)





	def    __UpperCAmelCase							(     self		,       __a							):

						'''simple docstring'''





						__a    : str      					=						list(self.config.features							) if self.config.features is not None else ['text']
						for file_idx, file in enumerate(itertools.chain.from_iterable(__a							)							):
											# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
											with open(__a		,       encoding=self.config.encoding		,       errors=self.config.errors							) as f:
																if self.config.sample_by == "line":
																					__a    : List[Any]      					=						0
																					while True:
																										__a    : Union[str, Any]      					=						f.read(self.config.chunksize							)
																										if not batch:
																															break
																										batch += f.readline()  # finish current line
																										# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																										__a    : int      					=						StringIO(__a							).readlines()
																										if not self.config.keep_linebreaks:
																															__a    : int      					=						[line.rstrip('\n'							) for line in batch]
																										__a    : str      					=						pa.Table.from_arrays([pa.array(__a							)]		,       names=__a							)
																										# Uncomment for debugging (will print the Arrow table size and elements)
																										# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																										# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																										yield (file_idx, batch_idx), self._cast_table(__a							)
																										batch_idx += 1
																elif self.config.sample_by == "paragraph":
																					__a    : List[Any]      					=						0
																					__a    : int      					=						''
																					while True:
																										__a    : Tuple      					=						f.read(self.config.chunksize							)
																										if not new_batch:
																															break
																										batch += new_batch
																										batch += f.readline()  # finish current line
																										__a    : str      					=						batch.split('\n\n'							)
																										__a    : Optional[Any]      					=						pa.Table.from_arrays(
																										    [pa.array([example for example in batch[:-1] if example]							)]		,       names=__a							)
																										# Uncomment for debugging (will print the Arrow table size and elements)
																										# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																										# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																										yield (file_idx, batch_idx), self._cast_table(__a							)
																										batch_idx += 1
																										__a    : Union[str, Any]      					=						batch[-1]
																					if batch:
																										__a    : Tuple      					=						pa.Table.from_arrays([pa.array([batch]							)]		,       names=__a							)
																										yield (file_idx, batch_idx), self._cast_table(__a							)
																elif self.config.sample_by == "document":
																					__a    : Dict      					=						f.read()
																					__a    : Any      					=						pa.Table.from_arrays([pa.array([text]							)]		,       names=__a							)
																					yield file_idx, self._cast_table(__a							)







'''simple docstring'''



def 						lowerCamelCase							(_SCREAMING_SNAKE_CASE   :     set		,   _SCREAMING_SNAKE_CASE   :     set     ):
					assert isinstance(_SCREAMING_SNAKE_CASE		,   _SCREAMING_SNAKE_CASE     ), F"""The input value of [set_a={set_a}] is not a set"""
					assert isinstance(_SCREAMING_SNAKE_CASE		,   _SCREAMING_SNAKE_CASE     ), F"""The input value of [set_b={set_b}] is not a set"""

					return {a + b for a in set_a for b in set_b}


if __name__ == "__main__":
	from doctest import testmod

	testmod()



