






import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


UpperCAmelCase__			 =	datasets.utils.logging.get_logger(__name__)




@dataclass
class        snake_case_					(						datasets.BuilderConfig			):

						"""simple docstring"""



						snake_case__   					=			None




class        snake_case_					(						datasets.ArrowBasedBuilder			):

						"""simple docstring"""



						snake_case__   					=			ArrowConfig



						def 					UpperCAmelCase__				(self:				Dict      )						->    Union[str, Any]:




											'''simple docstring'''



											return datasets.DatasetInfo(features=self.config.features      )



						def 					UpperCAmelCase__				(self:				Any   ,				__UpperCAmelCase:				Optional[Any]      )						->    List[Any]:




											'''simple docstring'''



											if not self.config.data_files:
																raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'      )
											__a				:	Union[str, Any]			=				dl_manager.download_and_extract(self.config.data_files      )
											if isinstance(__UpperCAmelCase   ,				(str, list, tuple)      ):
																__a				:	Optional[Any]			=				data_files
																if isinstance(__UpperCAmelCase   ,				__UpperCAmelCase      ):
																					__a				:	List[str]			=				[files]
																# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																__a				:	Any			=				[dl_manager.iter_files(__UpperCAmelCase      ) for file in files]
																return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,				gen_kwargs={"files": files}      )]
											__a				:	Any			=				[]
											for split_name, files in data_files.items():
																if isinstance(__UpperCAmelCase   ,				__UpperCAmelCase      ):
																					__a				:	Optional[Any]			=				[files]
																# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																__a				:	Optional[int]			=				[dl_manager.iter_files(__UpperCAmelCase      ) for file in files]
																# Infer features is they are stoed in the arrow schema
																if self.info.features is None:
																					for file in itertools.chain.from_iterable(__UpperCAmelCase      ):
																										with open(__UpperCAmelCase   ,				"rb"      ) as f:
																															__a				:	Optional[int]			=				datasets.Features.from_arrow_schema(pa.ipc.open_stream(__UpperCAmelCase      ).schema      )
																										break
																splits.append(datasets.SplitGenerator(name=__UpperCAmelCase   ,				gen_kwargs={"files": files}      )      )
											return splits



						def 					UpperCAmelCase__				(self:				str   ,				__UpperCAmelCase:				pa.Table      )						->    pa.Table:




											'''simple docstring'''



											if self.info.features is not None:
																# more expensive cast to support nested features with keys in a different order
																# allows str <-> int/float or str to Audio for example
																__a				:	Dict			=				table_cast(__UpperCAmelCase   ,				self.info.features.arrow_schema      )
											return pa_table



						def 					UpperCAmelCase__				(self:				Optional[Any]   ,				__UpperCAmelCase:				str      )						->    List[str]:




											'''simple docstring'''



											for file_idx, file in enumerate(itertools.chain.from_iterable(__UpperCAmelCase      )      ):
																with open(__UpperCAmelCase   ,				"rb"      ) as f:
																					try:
																										for batch_idx, record_batch in enumerate(pa.ipc.open_stream(__UpperCAmelCase      )      ):
																															__a				:	int			=				pa.Table.from_batches([record_batch]      )
																															# Uncomment for debugging (will print the Arrow table size and elements)
																															# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																															# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																															yield f'{file_idx}_{batch_idx}', self._cast_table(__UpperCAmelCase      )
																					except ValueError as e:
																										logger.error(f'Failed to read file \'{file}\' with error {type(__UpperCAmelCase      )}: {e}'      )
																										raise













from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase__			 =	{'''configuration_pegasus''': ['''PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''PegasusConfig''']}

try:
			if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase__			 =	['''PegasusTokenizer''']

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase__			 =	['''PegasusTokenizerFast''']

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase__			 =	[
			    '''PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''PegasusForCausalLM''',
			    '''PegasusForConditionalGeneration''',
			    '''PegasusModel''',
			    '''PegasusPreTrainedModel''',
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase__			 =	[
			    '''TFPegasusForConditionalGeneration''',
			    '''TFPegasusModel''',
			    '''TFPegasusPreTrainedModel''',
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase__			 =	[
			    '''FlaxPegasusForConditionalGeneration''',
			    '''FlaxPegasusModel''',
			    '''FlaxPegasusPreTrainedModel''',
			]


if TYPE_CHECKING:
			from .configuration_pegasus import PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP, PegasusConfig

			try:
						if not is_sentencepiece_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_pegasus import PegasusTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_pegasus_fast import PegasusTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_pegasus import (
						    PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST,
						    PegasusForCausalLM,
						    PegasusForConditionalGeneration,
						    PegasusModel,
						    PegasusPreTrainedModel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_pegasus import TFPegasusForConditionalGeneration, TFPegasusModel, TFPegasusPreTrainedModel

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_flax_pegasus import (
						    FlaxPegasusForConditionalGeneration,
						    FlaxPegasusModel,
						    FlaxPegasusPreTrainedModel,
						)

else:
			import sys

			UpperCAmelCase__			 =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





