






'''simple docstring'''







import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


_A      :     Optional[Any]       	=datasets.utils.logging.get_logger(__name__)

@dataclass
class _lowercase      (							datasets.BuilderConfig						):
							a							 =			None



class _lowercase      (							datasets.ArrowBasedBuilder						):
							a							 =			ArrowConfig




							def 						lowerCamelCase_							(				self:							int	):
														return datasets.DatasetInfo(features=self.config.features	)




							def 						lowerCamelCase_							(				self:							str     ,				UpperCamelCase__:							List[str]	):
														if not self.config.data_files:
																					raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''	)
														lowerCamelCase__   :      Tuple			    =			dl_manager.download_and_extract(self.config.data_files	)
														if isinstance(UpperCamelCase__     ,				(str, list, tuple)	):
																					lowerCamelCase__   :      List[Any]			    =			data_files
																					if isinstance(UpperCamelCase__     ,				UpperCamelCase__	):
																												lowerCamelCase__   :      Dict			    =			[files]
																					# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																					lowerCamelCase__   :      Dict			    =			[dl_manager.iter_files(UpperCamelCase__	) for file in files]
																					return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,				gen_kwargs={"""files""": files}	)]
														lowerCamelCase__   :      Any			    =			[]
														for split_name, files in data_files.items():
																					if isinstance(UpperCamelCase__     ,				UpperCamelCase__	):
																												lowerCamelCase__   :      Optional[Any]			    =			[files]
																					# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																					lowerCamelCase__   :      List[Any]			    =			[dl_manager.iter_files(UpperCamelCase__	) for file in files]
																					# Infer features is they are stoed in the arrow schema
																					if self.info.features is None:
																												for file in itertools.chain.from_iterable(UpperCamelCase__	):
																																			with open(UpperCamelCase__     ,				"""rb"""	) as f:
																																										lowerCamelCase__   :      List[Any]			    =			datasets.Features.from_arrow_schema(pa.ipc.open_stream(UpperCamelCase__	).schema	)
																																			break
																					splits.append(datasets.SplitGenerator(name=UpperCamelCase__     ,				gen_kwargs={"""files""": files}	)	)
														return splits




							def 						lowerCamelCase_							(				self:							Tuple     ,				UpperCamelCase__:							pa.Table	):
														if self.info.features is not None:
																					# more expensive cast to support nested features with keys in a different order
																					# allows str <-> int/float or str to Audio for example
																					lowerCamelCase__   :      Union[str, Any]			    =			table_cast(UpperCamelCase__     ,				self.info.features.arrow_schema	)
														return pa_table




							def 						lowerCamelCase_							(				self:							Union[str, Any]     ,				UpperCamelCase__:							int	):
														for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase__	)	):
																					with open(UpperCamelCase__     ,				"""rb"""	) as f:
																												try:
																																			for batch_idx, record_batch in enumerate(pa.ipc.open_stream(UpperCamelCase__	)	):
																																										lowerCamelCase__   :      str			    =			pa.Table.from_batches([record_batch]	)
																																										# Uncomment for debugging (will print the Arrow table size and elements)
																																										# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																										# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																										yield F'''{file_idx}_{batch_idx}''', self._cast_table(UpperCamelCase__	)
																												except ValueError as e:
																																			logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase__	)}: {e}'''	)
																																			raise









'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_A      :     Optional[Any]       	={
    '''configuration_realm''': ['''REALM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''RealmConfig'''],
    '''tokenization_realm''': ['''RealmTokenizer'''],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A      :     str       	=['''RealmTokenizerFast''']

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A      :     List[Any]       	=[
			    '''REALM_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''RealmEmbedder''',
			    '''RealmForOpenQA''',
			    '''RealmKnowledgeAugEncoder''',
			    '''RealmPreTrainedModel''',
			    '''RealmReader''',
			    '''RealmScorer''',
			    '''load_tf_weights_in_realm''',
			]
			_A      :     Union[str, Any]       	=['''RealmRetriever''']


if TYPE_CHECKING:
			from .configuration_realm import REALM_PRETRAINED_CONFIG_ARCHIVE_MAP, RealmConfig
			from .tokenization_realm import RealmTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_realm import RealmTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_realm import (
						    REALM_PRETRAINED_MODEL_ARCHIVE_LIST,
						    RealmEmbedder,
						    RealmForOpenQA,
						    RealmKnowledgeAugEncoder,
						    RealmPreTrainedModel,
						    RealmReader,
						    RealmScorer,
						    load_tf_weights_in_realm,
						)
						from .retrieval_realm import RealmRetriever


else:
			import sys

			_A      :     int       	=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

