






'''simple docstring'''







import os
import tempfile
from pathlib import Path
from unittest import TestCase

import pyarrow as pa
import pytest

from datasets.arrow_dataset import Dataset
from datasets.arrow_reader import ArrowReader, BaseReader, FileInstructions, ReadInstruction, make_file_instructions
from datasets.info import DatasetInfo
from datasets.splits import NamedSplit, Split, SplitDict, SplitInfo

from .utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




class 					__UpperCAmelCase (		UpperCamelCase__	):

      '''simple docstring'''





      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase  ,		_UpperCAmelCase=False						):
             UpperCAmelCase__							,							UpperCAmelCase__							,							UpperCAmelCase__	:   List[str]												=	(
                 filename_skip_take['''filename'''],
                 filename_skip_take['''skip'''] if '''skip''' in filename_skip_take else None,
                 filename_skip_take['''take'''] if '''take''' in filename_skip_take else None,
             )
             open(os.path.join(_UpperCAmelCase						)  ,		'''wb'''						).close()
             UpperCAmelCase__	:   str												=	pa.Table.from_pydict({'''filename''': [Path(_UpperCAmelCase						).name] * 100}						)
             if take == -1:
                    UpperCAmelCase__	:   Tuple												=	len(_UpperCAmelCase						) - skip
             if skip is not None and take is not None:
                    UpperCAmelCase__	:   Tuple												=	pa_table.slice(_UpperCAmelCase  ,		_UpperCAmelCase						)
             return pa_table




class 					__UpperCAmelCase (		UpperCamelCase__	):

      '''simple docstring'''





      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   List[str]												=	'''my_name'''
             UpperCAmelCase__	:   Any												=	SplitInfo(name='''train'''  ,		num_examples=100						)
             UpperCAmelCase__	:   List[str]												=	SplitInfo(name='''test'''  ,		num_examples=100						)
             UpperCAmelCase__	:   Union[str, Any]												=	[train_info, test_info]
             UpperCAmelCase__	:   Optional[int]												=	SplitDict()
             split_dict.add(_UpperCAmelCase						)
             split_dict.add(_UpperCAmelCase						)
             UpperCAmelCase__	:   Dict												=	DatasetInfo(splits=_UpperCAmelCase						)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    UpperCAmelCase__	:   Tuple												=	ReaderTest(_UpperCAmelCase  ,		_UpperCAmelCase						)

                    UpperCAmelCase__	:   Optional[Any]												=	'''test[:33%]'''
                    UpperCAmelCase__	:   List[str]												=	Dataset(**reader.read(_UpperCAmelCase  ,		_UpperCAmelCase  ,		_UpperCAmelCase						)						)
                    self.assertEqual(dset['''filename'''][0]  ,		F"""{name}-test"""						)
                    self.assertEqual(dset.num_rows  ,		33						)
                    self.assertEqual(dset.num_columns  ,		1						)

                    UpperCAmelCase__	:   Union[str, Any]												=	['''train''', '''test[:33%]''']
                    UpperCAmelCase__	:   List[Any]												=	[Split.TRAIN, ReadInstruction.from_spec('''test[:33%]'''						)]
                    for instructions in [instructionsa, instructionsa]:
                           UpperCAmelCase__	:   List[Any]												=	[reader.read(_UpperCAmelCase  ,		_UpperCAmelCase  ,		_UpperCAmelCase						) for instr in instructions]
                           UpperCAmelCase__							,							UpperCAmelCase__	:   Dict												=	(Dataset(**_UpperCAmelCase						) for dataset_kwargs in datasets_kwargs)
                           self.assertEqual(train_dset['''filename'''][0]  ,		F"""{name}-train"""						)
                           self.assertEqual(train_dset.num_rows  ,		100						)
                           self.assertEqual(train_dset.num_columns  ,		1						)
                           self.assertIsInstance(train_dset.split  ,		_UpperCAmelCase						)
                           self.assertEqual(str(train_dset.split						)  ,		'''train'''						)
                           self.assertEqual(test_dset['''filename'''][0]  ,		F"""{name}-test"""						)
                           self.assertEqual(test_dset.num_rows  ,		33						)
                           self.assertEqual(test_dset.num_columns  ,		1						)
                           self.assertIsInstance(test_dset.split  ,		_UpperCAmelCase						)
                           self.assertEqual(str(test_dset.split						)  ,		'''test[:33%]'''						)
                           del train_dset, test_dset




      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Optional[int]												=	'''my_name'''
             UpperCAmelCase__	:   Any												=	SplitInfo(name='''train'''  ,		num_examples=1000  ,		shard_lengths=[100] * 10						)
             UpperCAmelCase__	:   Union[str, Any]												=	[train_info]
             UpperCAmelCase__	:   Any												=	SplitDict()
             split_dict.add(_UpperCAmelCase						)
             UpperCAmelCase__	:   List[str]												=	DatasetInfo(splits=_UpperCAmelCase						)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    UpperCAmelCase__	:   Optional[Any]												=	ReaderTest(_UpperCAmelCase  ,		_UpperCAmelCase						)

                    UpperCAmelCase__	:   Tuple												=	'''train[:33%]'''
                    UpperCAmelCase__	:   Dict												=	Dataset(**reader.read(_UpperCAmelCase  ,		_UpperCAmelCase  ,		_UpperCAmelCase						)						)
                    self.assertEqual(dset['''filename'''][0]  ,		F"""{name}-train-00000-of-00010"""						)
                    self.assertEqual(dset['''filename'''][-1]  ,		F"""{name}-train-00003-of-00010"""						)
                    self.assertEqual(dset.num_rows  ,		330						)
                    self.assertEqual(dset.num_columns  ,		1						)




      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Any												=	SplitInfo(name='''train'''  ,		num_examples=100						)
             UpperCAmelCase__	:   str												=	SplitInfo(name='''test'''  ,		num_examples=100						)
             UpperCAmelCase__	:   Optional[Any]												=	SplitDict()
             split_dict.add(_UpperCAmelCase						)
             split_dict.add(_UpperCAmelCase						)
             UpperCAmelCase__	:   List[Any]												=	DatasetInfo(splits=_UpperCAmelCase						)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    UpperCAmelCase__	:   Optional[int]												=	ReaderTest(_UpperCAmelCase  ,		_UpperCAmelCase						)

                    UpperCAmelCase__	:   Union[str, Any]												=	[
                        {'''filename''': os.path.join(_UpperCAmelCase  ,		'''train'''						)},
                        {'''filename''': os.path.join(_UpperCAmelCase  ,		'''test'''						), '''skip''': 10, '''take''': 10},
                    ]
                    UpperCAmelCase__	:   Any												=	Dataset(**reader.read_files(_UpperCAmelCase  ,		original_instructions='''train+test[10:20]'''						)						)
                    self.assertEqual(dset.num_rows  ,		110						)
                    self.assertEqual(dset.num_columns  ,		1						)
                    del dset






@pytest.mark.parametrize('''in_memory'''			,						[False, True]			)
def  lowerCAmelCase__		(					a_  :     Tuple			,						a_  :     Union[str, Any]			,						a_  :     List[Any]			)   -> List[str]:
       UpperCAmelCase__	:   Any												=	arrow_file
       with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
              UpperCAmelCase__	:   List[str]												=	ArrowReader.read_table(a_			,						in_memory=a_			)
       assert table.shape == dataset.data.shape
       assert set(table.column_names			) == set(dataset.data.column_names			)
       assert dict(table.to_pydict()			) == dict(dataset.data.to_pydict()			)  # to_pydict returns OrderedDict






@pytest.mark.parametrize('''in_memory'''			,						[False, True]			)
def  lowerCAmelCase__		(					a_  :     Optional[Any]			,						a_  :     int			,						a_  :     List[str]			)   -> Tuple:
       UpperCAmelCase__	:   List[str]												=	arrow_file
       UpperCAmelCase__	:   Tuple												=	ArrowReader(''''''			,						a_			)
       with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
              UpperCAmelCase__	:   Tuple												=	reader.read_files([{'''filename''': filename}]			,						in_memory=a_			)
       assert dataset_kwargs.keys() == {"arrow_table", "info", "split"}
       UpperCAmelCase__	:   Any												=	dataset_kwargs['''arrow_table''']
       assert table.shape == dataset.data.shape
       assert set(table.column_names			) == set(dataset.data.column_names			)
       assert dict(table.to_pydict()			) == dict(dataset.data.to_pydict()			)  # to_pydict returns OrderedDict






def  lowerCAmelCase__		(					)   -> int:
       assert ReadInstruction('''train'''			,						to=1_0			,						unit='''abs'''			).to_spec() == "train[:10]"
       assert ReadInstruction('''train'''			,						from_=-8_0			,						to=1_0			,						unit='''%'''			).to_spec() == "train[-80%:10%]"

       UpperCAmelCase__	:   Any												=	'''train+test'''
       assert ReadInstruction.from_spec(a_			).to_spec() == spec_train_test

       UpperCAmelCase__	:   List[Any]												=	'''train[2:10]'''
       assert ReadInstruction.from_spec(a_			).to_spec() == spec_train_abs

       UpperCAmelCase__	:   List[str]												=	'''train[15%:-20%]'''
       assert ReadInstruction.from_spec(a_			).to_spec() == spec_train_pct

       UpperCAmelCase__	:   List[Any]												=	'''train[:10%](closest)'''
       assert ReadInstruction.from_spec(a_			).to_spec() == "train[:10%]"

       UpperCAmelCase__	:   List[str]												=	'''train[:10%](pct1_dropremainder)'''
       assert ReadInstruction.from_spec(a_			).to_spec() == spec_train_pct_rounding

       UpperCAmelCase__	:   List[str]												=	'''train[:10%](pct1_dropremainder)+test[-10%:](pct1_dropremainder)'''
       assert ReadInstruction.from_spec(a_			).to_spec() == spec_train_test_pct_rounding






def  lowerCAmelCase__		(					)   -> List[str]:
       UpperCAmelCase__	:   Optional[Any]												=	'''dummy'''
       UpperCAmelCase__	:   Tuple												=	[SplitInfo(name='''train'''			,						num_examples=1_0_0			)]
       UpperCAmelCase__	:   str												=	'''train[:33%]'''
       UpperCAmelCase__	:   Optional[int]												=	'''arrow'''
       UpperCAmelCase__	:   Optional[Any]												=	'''prefix'''

       UpperCAmelCase__	:   str												=	make_file_instructions(a_			,						a_			,						a_			,						a_			,						a_			)
       assert isinstance(a_			,						a_			)
       assert file_instructions.num_examples == 3_3
       assert file_instructions.file_instructions == [
           {"filename": os.path.join(a_			,						f"""{name}-train.arrow"""			), "skip": 0, "take": 3_3}
       ]

       UpperCAmelCase__	:   Dict												=	[SplitInfo(name='''train'''			,						num_examples=1_0_0			,						shard_lengths=[1_0] * 1_0			)]
       UpperCAmelCase__	:   Tuple												=	make_file_instructions(a_			,						a_			,						a_			,						a_			,						a_			)
       assert isinstance(a_			,						a_			)
       assert file_instructions.num_examples == 3_3
       assert file_instructions.file_instructions == [
           {"filename": os.path.join(a_			,						f"""{name}-train-00000-of-00010.arrow"""			), "skip": 0, "take": -1},
           {"filename": os.path.join(a_			,						f"""{name}-train-00001-of-00010.arrow"""			), "skip": 0, "take": -1},
           {"filename": os.path.join(a_			,						f"""{name}-train-00002-of-00010.arrow"""			), "skip": 0, "take": -1},
           {"filename": os.path.join(a_			,						f"""{name}-train-00003-of-00010.arrow"""			), "skip": 0, "take": 3},
       ]






@pytest.mark.parametrize('''name, expected_exception'''			,						[(None, TypeError), ('''''', ValueError)]			)
def  lowerCAmelCase__		(					a_  :     str			,						a_  :     List[str]			)   -> Any:
       UpperCAmelCase__	:   int												=	[SplitInfo(name='''train'''			,						num_examples=1_0_0			)]
       UpperCAmelCase__	:   List[Any]												=	'''train'''
       UpperCAmelCase__	:   Tuple												=	'''arrow'''
       UpperCAmelCase__	:   Union[str, Any]												=	'''prefix_path'''
       with pytest.raises(a_			):
              UpperCAmelCase__	:   Any												=	make_file_instructions(a_			,						a_			,						a_			,						a_			,						a_			)







'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCamelCase_					=				{
    "configuration_flava": [
        "FLAVA_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "FlavaConfig",
        "FlavaImageCodebookConfig",
        "FlavaImageConfig",
        "FlavaMultimodalConfig",
        "FlavaTextConfig",
    ],
}

try:
  if not is_vision_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCamelCase_					=				["FlavaFeatureExtractor"]
  UpperCamelCase_					=				["FlavaImageProcessor"]
  UpperCamelCase_					=				["FlavaProcessor"]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCamelCase_					=				[
      "FLAVA_PRETRAINED_MODEL_ARCHIVE_LIST",
      "FlavaForPreTraining",
      "FlavaImageCodebook",
      "FlavaImageModel",
      "FlavaModel",
      "FlavaMultimodalModel",
      "FlavaPreTrainedModel",
      "FlavaTextModel",
  ]

if TYPE_CHECKING:
  from .configuration_flava import (
      FLAVA_PRETRAINED_CONFIG_ARCHIVE_MAP,
      FlavaConfig,
      FlavaImageCodebookConfig,
      FlavaImageConfig,
      FlavaMultimodalConfig,
      FlavaTextConfig,
  )

  try:
    if not is_vision_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .feature_extraction_flava import FlavaFeatureExtractor
    from .image_processing_flava import FlavaImageProcessor
    from .processing_flava import FlavaProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flava import (
        FLAVA_PRETRAINED_MODEL_ARCHIVE_LIST,
        FlavaForPreTraining,
        FlavaImageCodebook,
        FlavaImageModel,
        FlavaModel,
        FlavaMultimodalModel,
        FlavaPreTrainedModel,
        FlavaTextModel,
    )

else:
  import sys

  UpperCamelCase_					=				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)