
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_A	  =					{
    '''configuration_deta''': ['''DETA_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''DetaConfig'''],
}

try:
      if not is_vision_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _A	  =					['''DetaImageProcessor''']

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _A	  =					[
          '''DETA_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''DetaForObjectDetection''',
          '''DetaModel''',
          '''DetaPreTrainedModel''',
      ]


if TYPE_CHECKING:
      from .configuration_deta import DETA_PRETRAINED_CONFIG_ARCHIVE_MAP, DetaConfig

      try:
            if not is_vision_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .image_processing_deta import DetaImageProcessor

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_deta import (
                DETA_PRETRAINED_MODEL_ARCHIVE_LIST,
                DetaForObjectDetection,
                DetaModel,
                DetaPreTrainedModel,
            )

else:
      import sys

      _A	  =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_A	  =					datasets.utils.logging.get_logger(__name__)

@dataclass
class 		lowercase_       (       datasets.BuilderConfig				):
 A__						:				Optional[datasets.Features]         =    None
 A__						:				str         =    "utf-8"
 A__						:				Optional[str]         =    None
 A__						:				int         =    10 << 20  # 10MB
 A__						:				bool         =    False
 A__						:				str         =    "line"



class 		lowercase_       (       datasets.ArrowBasedBuilder				):
 A__						:				str         =    TextConfig

 def lowerCamelCase_				(      self     ):




        """simple docstring"""


        return datasets.DatasetInfo(features=self.config.features     )

 def lowerCamelCase_				(      self     ,	__UpperCamelCase     ):




        """simple docstring"""


        if not self.config.data_files:
               raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''     )
        UpperCamelCase_							       =       dl_manager.download_and_extract(self.config.data_files     )
        if isinstance(__UpperCamelCase     ,	(str, list, tuple)     ):
               UpperCamelCase_							       =       data_files
               if isinstance(__UpperCamelCase     ,	__UpperCamelCase     ):
                      UpperCamelCase_							       =       [files]
               UpperCamelCase_							       =       [dl_manager.iter_files(__UpperCamelCase     ) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,	gen_kwargs={"""files""": files}     )]
        UpperCamelCase_							       =       []
        for split_name, files in data_files.items():
               if isinstance(__UpperCamelCase     ,	__UpperCamelCase     ):
                      UpperCamelCase_							       =       [files]
               UpperCamelCase_							       =       [dl_manager.iter_files(__UpperCamelCase     ) for file in files]
               splits.append(datasets.SplitGenerator(name=__UpperCamelCase     ,	gen_kwargs={"""files""": files}     )     )
        return splits

 def lowerCamelCase_				(      self     ,	__UpperCamelCase     ):




        """simple docstring"""


        if self.config.features is not None:
               UpperCamelCase_							       =       self.config.features.arrow_schema
               if all(not require_storage_cast(__UpperCamelCase     ) for feature in self.config.features.values()     ):
                      # cheaper cast
                      UpperCamelCase_							       =       pa_table.cast(__UpperCamelCase     )
               else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      UpperCamelCase_							       =       table_cast(__UpperCamelCase     ,	__UpperCamelCase     )
               return pa_table
        else:
               return pa_table.cast(pa.schema({"""text""": pa.string()}     )     )

 def lowerCamelCase_				(      self     ,	__UpperCamelCase     ):




        """simple docstring"""


        UpperCamelCase_							       =       list(self.config.features     ) if self.config.features is not None else ["""text"""]
        for file_idx, file in enumerate(itertools.chain.from_iterable(__UpperCamelCase     )     ):
               # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
               with open(__UpperCamelCase     ,	encoding=self.config.encoding     ,	errors=self.config.errors     ) as f:
                      if self.config.sample_by == "line":
                             UpperCamelCase_							       =       0
                             while True:
                                    UpperCamelCase_							       =       f.read(self.config.chunksize     )
                                    if not batch:
                                           break
                                    batch += f.readline()  # finish current line
                                    # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                    UpperCamelCase_							       =       StringIO(__UpperCamelCase     ).readlines()
                                    if not self.config.keep_linebreaks:
                                           UpperCamelCase_							       =       [line.rstrip("""\n"""     ) for line in batch]
                                    UpperCamelCase_							       =       pa.Table.from_arrays([pa.array(__UpperCamelCase     )]     ,	names=__UpperCamelCase     )
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield (file_idx, batch_idx), self._cast_table(__UpperCamelCase     )
                                    batch_idx += 1
                      elif self.config.sample_by == "paragraph":
                             UpperCamelCase_							       =       0
                             UpperCamelCase_							       =       """"""
                             while True:
                                    UpperCamelCase_							       =       f.read(self.config.chunksize     )
                                    if not new_batch:
                                           break
                                    batch += new_batch
                                    batch += f.readline()  # finish current line
                                    UpperCamelCase_							       =       batch.split("""\n\n"""     )
                                    UpperCamelCase_							       =       pa.Table.from_arrays(
                                        [pa.array([example for example in batch[:-1] if example]     )]     ,	names=__UpperCamelCase     )
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield (file_idx, batch_idx), self._cast_table(__UpperCamelCase     )
                                    batch_idx += 1
                                    UpperCamelCase_							       =       batch[-1]
                             if batch:
                                    UpperCamelCase_							       =       pa.Table.from_arrays([pa.array([batch]     )]     ,	names=__UpperCamelCase     )
                                    yield (file_idx, batch_idx), self._cast_table(__UpperCamelCase     )
                      elif self.config.sample_by == "document":
                             UpperCamelCase_							       =       f.read()
                             UpperCamelCase_							       =       pa.Table.from_arrays([pa.array([text]     )]     ,	names=__UpperCamelCase     )
                             yield file_idx, self._cast_table(__UpperCamelCase     )



