'''simple docstring'''




import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json




@pytest.fixture
def 				lowerCamelCase      ( UpperCAmelCase__			:   str					)     -> List[str]:
							lowercase_   :     List[Any]									=			tmp_path / """file.jsonl"""
							lowercase_   :     List[Any]									=			textwrap.dedent(
							    """\
        {\"col_1\": -1}
        {\"col_1\": 1, \"col_2\": 2}
        {\"col_1\": 10, \"col_2\": 20}
        """					)
							with open(UpperCAmelCase__ , """w"""					) as f:
														f.write(UpperCAmelCase__					)
							return str(UpperCAmelCase__					)




@pytest.fixture
def 				lowerCamelCase      ( UpperCAmelCase__			:   Tuple					)     -> Tuple:
							lowercase_   :     List[Any]									=			tmp_path / """file_utf16_encoded.jsonl"""
							lowercase_   :     Any									=			textwrap.dedent(
							    """\
        {\"col_1\": -1}
        {\"col_1\": 1, \"col_2\": 2}
        {\"col_1\": 10, \"col_2\": 20}
        """					)
							with open(UpperCAmelCase__ , """w""" , encoding="""utf-16"""					) as f:
														f.write(UpperCAmelCase__					)
							return str(UpperCAmelCase__					)




@pytest.fixture
def 				lowerCamelCase      ( UpperCAmelCase__			:   Tuple					)     -> Tuple:
							lowercase_   :     Optional[int]									=			tmp_path / """file_with_list_of_dicts.json"""
							lowercase_   :     Tuple									=			textwrap.dedent(
							    """\
        [
            {\"col_1\": -1},
            {\"col_1\": 1, \"col_2\": 2},
            {\"col_1\": 10, \"col_2\": 20}
        ]
        """					)
							with open(UpperCAmelCase__ , """w"""					) as f:
														f.write(UpperCAmelCase__					)
							return str(UpperCAmelCase__					)




@pytest.fixture
def 				lowerCamelCase      ( UpperCAmelCase__			:   Any					)     -> Optional[Any]:
							lowercase_   :     List[str]									=			tmp_path / """file_with_list_of_dicts_field.json"""
							lowercase_   :     Tuple									=			textwrap.dedent(
							    """\
        {
            \"field1\": 1,
            \"field2\": \"aabb\",
            \"field3\": [
                {\"col_1\": -1},
                {\"col_1\": 1, \"col_2\": 2},
                {\"col_1\": 10, \"col_2\": 20}
            ]
        }
        """					)
							with open(UpperCAmelCase__ , """w"""					) as f:
														f.write(UpperCAmelCase__					)
							return str(UpperCAmelCase__					)




@pytest.mark.parametrize(
    """file_fixture, config_kwargs""" , [
        ("""jsonl_file""", {}),
        ("""jsonl_file_utf16_encoded""", {"""encoding""": """utf-16"""}),
        ("""json_file_with_list_of_dicts""", {}),
        ("""json_file_with_list_of_dicts_field""", {"""field""": """field3"""}),
    ] , )
def 				lowerCamelCase      ( UpperCAmelCase__			:   Optional[Any] , UpperCAmelCase__			:   str , UpperCAmelCase__			:   Optional[Any]					)     -> Union[str, Any]:
							lowercase_   :     Tuple									=			Json(**UpperCAmelCase__					)
							lowercase_   :     Dict									=			json._generate_tables([[request.getfixturevalue(UpperCAmelCase__					)]]					)
							lowercase_   :     Tuple									=			pa.concat_tables([table for _, table in generator]					)
							assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}




@pytest.mark.parametrize(
    """file_fixture, config_kwargs""" , [
        (
            """jsonl_file""",
            {"""features""": Features({"""col_1""": Value("""int64"""					), """col_2""": Value("""int64"""					), """missing_col""": Value("""string"""					)}					)},
        ),
        (
            """json_file_with_list_of_dicts""",
            {"""features""": Features({"""col_1""": Value("""int64"""					), """col_2""": Value("""int64"""					), """missing_col""": Value("""string"""					)}					)},
        ),
        (
            """json_file_with_list_of_dicts_field""",
            {
                """field""": """field3""",
                """features""": Features(
                    {"""col_1""": Value("""int64"""					), """col_2""": Value("""int64"""					), """missing_col""": Value("""string"""					)}					),
            },
        ),
    ] , )
def 				lowerCamelCase      ( UpperCAmelCase__			:   Tuple , UpperCAmelCase__			:   str , UpperCAmelCase__			:   Dict					)     -> Optional[Any]:
							lowercase_   :     Optional[Any]									=			Json(**UpperCAmelCase__					)
							lowercase_   :     int									=			json._generate_tables([[request.getfixturevalue(UpperCAmelCase__					)]]					)
							lowercase_   :     Tuple									=			pa.concat_tables([table for _, table in generator]					)
							assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}






'''simple docstring'''




import os
import tempfile
from pathlib import Path
from unittest import TestCase

import pyarrow as pa
import pytest

from datasets.arrow_dataset import Dataset
from datasets.arrow_reader import ArrowReader, BaseReader, FileInstructions, ReadInstruction, make_file_instructions
from datasets.info import DatasetInfo
from datasets.splits import NamedSplit, Split, SplitDict, SplitInfo

from .utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





class        __magic_name__		(       _UpperCAmelCase):







			def       SCREAMING_SNAKE_CASE_	(      self							:    Any							,     lowercase_							:    Dict							,     lowercase_							:    List[str]=False       ):
										lowercase_		,		lowercase_		,		lowercase_   :     str									=			(
										    filename_skip_take["""filename"""],
										    filename_skip_take["""skip"""] if """skip""" in filename_skip_take else None,
										    filename_skip_take["""take"""] if """take""" in filename_skip_take else None,
										)
										open(os.path.join(lowercase_       )							,     """wb"""       ).close()
										lowercase_   :     Union[str, Any]									=			pa.Table.from_pydict({"""filename""": [Path(lowercase_       ).name] * 100}       )
										if take == -1:
																	lowercase_   :     Optional[int]									=			len(lowercase_       ) - skip
										if skip is not None and take is not None:
																	lowercase_   :     Any									=			pa_table.slice(lowercase_							,     lowercase_       )
										return pa_table





class        __magic_name__		(       _UpperCAmelCase):







			def       SCREAMING_SNAKE_CASE_	(      self							:    int       ):
										lowercase_   :     Union[str, Any]									=			"""my_name"""
										lowercase_   :     Optional[Any]									=			SplitInfo(name="""train"""							,     num_examples=100       )
										lowercase_   :     Dict									=			SplitInfo(name="""test"""							,     num_examples=100       )
										lowercase_   :     str									=			[train_info, test_info]
										lowercase_   :     Any									=			SplitDict()
										split_dict.add(lowercase_       )
										split_dict.add(lowercase_       )
										lowercase_   :     Union[str, Any]									=			DatasetInfo(splits=lowercase_       )

										with tempfile.TemporaryDirectory() as tmp_dir:
																	lowercase_   :     Tuple									=			ReaderTest(lowercase_							,     lowercase_       )

																	lowercase_   :     Optional[Any]									=			"""test[:33%]"""
																	lowercase_   :     List[Any]									=			Dataset(**reader.read(lowercase_							,     lowercase_							,     lowercase_       )       )
																	self.assertEqual(dset["""filename"""][0]							,     f'''{name}-test'''       )
																	self.assertEqual(dset.num_rows							,     33       )
																	self.assertEqual(dset.num_columns							,     1       )

																	lowercase_   :     str									=			["""train""", """test[:33%]"""]
																	lowercase_   :     Any									=			[Split.TRAIN, ReadInstruction.from_spec("""test[:33%]"""       )]
																	for instructions in [instructionsa, instructionsa]:
																								lowercase_   :     Dict									=			[reader.read(lowercase_							,     lowercase_							,     lowercase_       ) for instr in instructions]
																								lowercase_		,		lowercase_   :     List[Any]									=			(Dataset(**lowercase_       ) for dataset_kwargs in datasets_kwargs)
																								self.assertEqual(train_dset["""filename"""][0]							,     f'''{name}-train'''       )
																								self.assertEqual(train_dset.num_rows							,     100       )
																								self.assertEqual(train_dset.num_columns							,     1       )
																								self.assertIsInstance(train_dset.split							,     lowercase_       )
																								self.assertEqual(str(train_dset.split       )							,     """train"""       )
																								self.assertEqual(test_dset["""filename"""][0]							,     f'''{name}-test'''       )
																								self.assertEqual(test_dset.num_rows							,     33       )
																								self.assertEqual(test_dset.num_columns							,     1       )
																								self.assertIsInstance(test_dset.split							,     lowercase_       )
																								self.assertEqual(str(test_dset.split       )							,     """test[:33%]"""       )
																								del train_dset, test_dset







			def       SCREAMING_SNAKE_CASE_	(      self							:    Dict       ):
										lowercase_   :     List[Any]									=			"""my_name"""
										lowercase_   :     int									=			SplitInfo(name="""train"""							,     num_examples=1000							,     shard_lengths=[100] * 10       )
										lowercase_   :     Tuple									=			[train_info]
										lowercase_   :     Any									=			SplitDict()
										split_dict.add(lowercase_       )
										lowercase_   :     Union[str, Any]									=			DatasetInfo(splits=lowercase_       )

										with tempfile.TemporaryDirectory() as tmp_dir:
																	lowercase_   :     Optional[int]									=			ReaderTest(lowercase_							,     lowercase_       )

																	lowercase_   :     Optional[int]									=			"""train[:33%]"""
																	lowercase_   :     int									=			Dataset(**reader.read(lowercase_							,     lowercase_							,     lowercase_       )       )
																	self.assertEqual(dset["""filename"""][0]							,     f'''{name}-train-00000-of-00010'''       )
																	self.assertEqual(dset["""filename"""][-1]							,     f'''{name}-train-00003-of-00010'''       )
																	self.assertEqual(dset.num_rows							,     330       )
																	self.assertEqual(dset.num_columns							,     1       )







			def       SCREAMING_SNAKE_CASE_	(      self							:    Any       ):
										lowercase_   :     Any									=			SplitInfo(name="""train"""							,     num_examples=100       )
										lowercase_   :     Optional[int]									=			SplitInfo(name="""test"""							,     num_examples=100       )
										lowercase_   :     Union[str, Any]									=			SplitDict()
										split_dict.add(lowercase_       )
										split_dict.add(lowercase_       )
										lowercase_   :     Tuple									=			DatasetInfo(splits=lowercase_       )

										with tempfile.TemporaryDirectory() as tmp_dir:
																	lowercase_   :     Tuple									=			ReaderTest(lowercase_							,     lowercase_       )

																	lowercase_   :     List[str]									=			[
																	    {"""filename""": os.path.join(lowercase_							,     """train"""       )},
																	    {"""filename""": os.path.join(lowercase_							,     """test"""       ), """skip""": 10, """take""": 10},
																	]
																	lowercase_   :     int									=			Dataset(**reader.read_files(lowercase_							,     original_instructions="""train+test[10:20]"""       )       )
																	self.assertEqual(dset.num_rows							,     110       )
																	self.assertEqual(dset.num_columns							,     1       )
																	del dset




@pytest.mark.parametrize("""in_memory""" , [False, True]					)
def 				lowerCamelCase      ( UpperCAmelCase__			:   List[Any] , UpperCAmelCase__			:   int , UpperCAmelCase__			:   Union[str, Any]					)     -> Union[str, Any]:
							lowercase_   :     str									=			arrow_file
							with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
														lowercase_   :     Optional[Any]									=			ArrowReader.read_table(UpperCAmelCase__ , in_memory=UpperCAmelCase__					)
							assert table.shape == dataset.data.shape
							assert set(table.column_names					) == set(dataset.data.column_names					)
							assert dict(table.to_pydict()					) == dict(dataset.data.to_pydict()					)  # to_pydict returns OrderedDict




@pytest.mark.parametrize("""in_memory""" , [False, True]					)
def 				lowerCamelCase      ( UpperCAmelCase__			:   Tuple , UpperCAmelCase__			:   int , UpperCAmelCase__			:   Optional[int]					)     -> Any:
							lowercase_   :     Tuple									=			arrow_file
							lowercase_   :     Optional[Any]									=			ArrowReader("""""" , UpperCAmelCase__					)
							with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
														lowercase_   :     Dict									=			reader.read_files([{"""filename""": filename}] , in_memory=UpperCAmelCase__					)
							assert dataset_kwargs.keys() == {"arrow_table", "info", "split"}
							lowercase_   :     Optional[int]									=			dataset_kwargs["""arrow_table"""]
							assert table.shape == dataset.data.shape
							assert set(table.column_names					) == set(dataset.data.column_names					)
							assert dict(table.to_pydict()					) == dict(dataset.data.to_pydict()					)  # to_pydict returns OrderedDict




def 				lowerCamelCase      ( )     -> Optional[Any]:
							assert ReadInstruction("""train""" , to=10 , unit="""abs"""					).to_spec() == "train[:10]"
							assert ReadInstruction("""train""" , from_=-80 , to=10 , unit="""%"""					).to_spec() == "train[-80%:10%]"

							lowercase_   :     Tuple									=			"""train+test"""
							assert ReadInstruction.from_spec(UpperCAmelCase__					).to_spec() == spec_train_test

							lowercase_   :     List[Any]									=			"""train[2:10]"""
							assert ReadInstruction.from_spec(UpperCAmelCase__					).to_spec() == spec_train_abs

							lowercase_   :     Any									=			"""train[15%:-20%]"""
							assert ReadInstruction.from_spec(UpperCAmelCase__					).to_spec() == spec_train_pct

							lowercase_   :     Optional[int]									=			"""train[:10%](closest)"""
							assert ReadInstruction.from_spec(UpperCAmelCase__					).to_spec() == "train[:10%]"

							lowercase_   :     List[Any]									=			"""train[:10%](pct1_dropremainder)"""
							assert ReadInstruction.from_spec(UpperCAmelCase__					).to_spec() == spec_train_pct_rounding

							lowercase_   :     Union[str, Any]									=			"""train[:10%](pct1_dropremainder)+test[-10%:](pct1_dropremainder)"""
							assert ReadInstruction.from_spec(UpperCAmelCase__					).to_spec() == spec_train_test_pct_rounding




def 				lowerCamelCase      ( )     -> int:
							lowercase_   :     Optional[int]									=			"""dummy"""
							lowercase_   :     Union[str, Any]									=			[SplitInfo(name="""train""" , num_examples=100					)]
							lowercase_   :     Optional[int]									=			"""train[:33%]"""
							lowercase_   :     Dict									=			"""arrow"""
							lowercase_   :     int									=			"""prefix"""

							lowercase_   :     int									=			make_file_instructions(UpperCAmelCase__ , UpperCAmelCase__ , UpperCAmelCase__ , UpperCAmelCase__ , UpperCAmelCase__					)
							assert isinstance(UpperCAmelCase__ , UpperCAmelCase__					)
							assert file_instructions.num_examples == 33
							assert file_instructions.file_instructions == [
							    {"filename": os.path.join(UpperCAmelCase__ , F'''{name}-train.arrow'''					), "skip": 0, "take": 33}
							]

							lowercase_   :     Dict									=			[SplitInfo(name="""train""" , num_examples=100 , shard_lengths=[10] * 10					)]
							lowercase_   :     Dict									=			make_file_instructions(UpperCAmelCase__ , UpperCAmelCase__ , UpperCAmelCase__ , UpperCAmelCase__ , UpperCAmelCase__					)
							assert isinstance(UpperCAmelCase__ , UpperCAmelCase__					)
							assert file_instructions.num_examples == 33
							assert file_instructions.file_instructions == [
							    {"filename": os.path.join(UpperCAmelCase__ , F'''{name}-train-00000-of-00010.arrow'''					), "skip": 0, "take": -1},
							    {"filename": os.path.join(UpperCAmelCase__ , F'''{name}-train-00001-of-00010.arrow'''					), "skip": 0, "take": -1},
							    {"filename": os.path.join(UpperCAmelCase__ , F'''{name}-train-00002-of-00010.arrow'''					), "skip": 0, "take": -1},
							    {"filename": os.path.join(UpperCAmelCase__ , F'''{name}-train-00003-of-00010.arrow'''					), "skip": 0, "take": 3},
							]




@pytest.mark.parametrize("""name, expected_exception""" , [(None, TypeError), ("""""", ValueError)]					)
def 				lowerCamelCase      ( UpperCAmelCase__			:   List[Any] , UpperCAmelCase__			:   Any					)     -> List[str]:
							lowercase_   :     str									=			[SplitInfo(name="""train""" , num_examples=100					)]
							lowercase_   :     Any									=			"""train"""
							lowercase_   :     Optional[int]									=			"""arrow"""
							lowercase_   :     str									=			"""prefix_path"""
							with pytest.raises(UpperCAmelCase__					):
														lowercase_   :     List[str]									=			make_file_instructions(UpperCAmelCase__ , UpperCAmelCase__ , UpperCAmelCase__ , UpperCAmelCase__ , UpperCAmelCase__					)





