





'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_a      :				Optional[Any]											=					{"""configuration_nat""": ["""NAT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """NatConfig"""]}


try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_a      :				Optional[int]											=					[
	    """NAT_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """NatForImageClassification""",
	    """NatModel""",
	    """NatPreTrainedModel""",
	    """NatBackbone""",
	]

if TYPE_CHECKING:
	from .configuration_nat import NAT_PRETRAINED_CONFIG_ARCHIVE_MAP, NatConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_nat import (
		    NAT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    NatBackbone,
		    NatForImageClassification,
		    NatModel,
		    NatPreTrainedModel,
		)

else:
	import sys

	_a      :				List[Any]											=					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)








'''simple docstring'''







import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_a      :				Tuple											=					datasets.utils.logging.get_logger(__name__)





@dataclass
class 					_UpperCAmelCase							(		datasets.BuilderConfig     ):
		a     :     Optional[datasets.Features]     						=None
		a     :     str     						="utf-8"
		a     :     Optional[str]     						=None
		a     :     int     						=10 << 20  # 10MB
		a     :     bool     						=False
		a     :     str     						="line"





class 					_UpperCAmelCase							(		datasets.ArrowBasedBuilder     ):
		a     :     Optional[int]     						=TextConfig
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					return datasets.DatasetInfo(features=self.config.features    )
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					if not self.config.data_files:
								raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'    )
					__lowerCAmelCase      		=       dl_manager.download_and_extract(self.config.data_files    )
					if isinstance(__SCREAMING_SNAKE_CASE,(str, list, tuple)    ):
								__lowerCAmelCase      		=       data_files
								if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ):
											__lowerCAmelCase      		=       [files]
								__lowerCAmelCase      		=       [dl_manager.iter_files(__SCREAMING_SNAKE_CASE    ) for file in files]
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN,gen_kwargs={"""files""": files}    )]
					__lowerCAmelCase      		=       []
					for split_name, files in data_files.items():
								if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ):
											__lowerCAmelCase      		=       [files]
								__lowerCAmelCase      		=       [dl_manager.iter_files(__SCREAMING_SNAKE_CASE    ) for file in files]
								splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE,gen_kwargs={"""files""": files}    )    )
					return splits
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					if self.config.features is not None:
								__lowerCAmelCase      		=       self.config.features.arrow_schema
								if all(not require_storage_cast(__SCREAMING_SNAKE_CASE    ) for feature in self.config.features.values()    ):
											# cheaper cast
											__lowerCAmelCase      		=       pa_table.cast(__SCREAMING_SNAKE_CASE    )
								else:
											# more expensive cast; allows str <-> int/float or str to Audio for example
											__lowerCAmelCase      		=       table_cast(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
								return pa_table
					else:
								return pa_table.cast(pa.schema({"""text""": pa.string()}    )    )







		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       list(self.config.features    ) if self.config.features is not None else ["""text"""]
					for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE    )    ):
								# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
								with open(__SCREAMING_SNAKE_CASE,encoding=self.config.encoding,errors=self.config.errors    ) as f:
											if self.config.sample_by == "line":
														__lowerCAmelCase      		=       0
														while True:
																	__lowerCAmelCase      		=       f.read(self.config.chunksize    )
																	if not batch:
																				break
																	batch += f.readline()  # finish current line
																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																	__lowerCAmelCase      		=       StringIO(__SCREAMING_SNAKE_CASE    ).readlines()
																	if not self.config.keep_linebreaks:
																				__lowerCAmelCase      		=       [line.rstrip("""\n"""    ) for line in batch]
																	__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array(__SCREAMING_SNAKE_CASE    )],names=__SCREAMING_SNAKE_CASE    )
																	# Uncomment for debugging (will print the Arrow table size and elements)
																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
																	batch_idx += 1
											elif self.config.sample_by == "paragraph":
														__lowerCAmelCase      		=       0
														__lowerCAmelCase      		=       """"""
														while True:
																	__lowerCAmelCase      		=       f.read(self.config.chunksize    )
																	if not new_batch:
																				break
																	batch += new_batch
																	batch += f.readline()  # finish current line
																	__lowerCAmelCase      		=       batch.split("""\n\n"""    )
																	__lowerCAmelCase      		=       pa.Table.from_arrays(
																	    [pa.array([example for example in batch[:-1] if example]    )],names=__SCREAMING_SNAKE_CASE    )
																	# Uncomment for debugging (will print the Arrow table size and elements)
																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
																	batch_idx += 1
																	__lowerCAmelCase      		=       batch[-1]
														if batch:
																	__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array([batch]    )],names=__SCREAMING_SNAKE_CASE    )
																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
											elif self.config.sample_by == "document":
														__lowerCAmelCase      		=       f.read()
														__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array([text]    )],names=__SCREAMING_SNAKE_CASE    )
														yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE    )

