





"""simple docstring"""
import os
import tempfile
from pathlib import Path
from unittest import TestCase

import pyarrow as pa
import pytest

from datasets.arrow_dataset import Dataset
from datasets.arrow_reader import ArrowReader, BaseReader, FileInstructions, ReadInstruction, make_file_instructions
from datasets.info import DatasetInfo
from datasets.splits import NamedSplit, Split, SplitDict, SplitInfo

from .utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
class lowerCamelCase__  (    lowerCamelCase_				):



   def    _UpperCAmelCase	(       self      ,   snake_case      ,   snake_case=False	) ->    List[str]:







      """simple docstring"""
      lowercase    :			List[Any]  =       (
          filename_skip_take["filename"],
          filename_skip_take["skip"] if "skip" in filename_skip_take else None,
          filename_skip_take["take"] if "take" in filename_skip_take else None,
      )
      open(os.path.join(snake_case	)      ,   """wb"""	).close()
      lowercase    :			Any  =       pa.Table.from_pydict({"""filename""": [Path(snake_case	).name] * 1_0_0}	)
      if take == -1:
         lowercase    :			Dict  =       len(snake_case	) - skip
      if skip is not None and take is not None:
         lowercase    :			Dict  =       pa_table.slice(snake_case      ,   snake_case	)
      return pa_table




class lowerCamelCase__  (    lowerCamelCase_				):



   def    _UpperCAmelCase	(       self	) ->    Optional[int]:







      """simple docstring"""
      lowercase    :			int  =       "my_name"
      lowercase    :			Union[str, Any]  =       SplitInfo(name="""train"""      ,   num_examples=1_0_0	)
      lowercase    :			Tuple  =       SplitInfo(name="""test"""      ,   num_examples=1_0_0	)
      lowercase    :			int  =       [train_info, test_info]
      lowercase    :			Dict  =       SplitDict()
      split_dict.add(snake_case	)
      split_dict.add(snake_case	)
      lowercase    :			Union[str, Any]  =       DatasetInfo(splits=snake_case	)

      with tempfile.TemporaryDirectory() as tmp_dir:
         lowercase    :			List[Any]  =       ReaderTest(snake_case      ,   snake_case	)

         lowercase    :			List[str]  =       "test[:33%]"
         lowercase    :			Tuple  =       Dataset(**reader.read(snake_case      ,   snake_case      ,   snake_case	)	)
         self.assertEqual(dset["""filename"""][0]      ,   f'''{name}-test'''	)
         self.assertEqual(dset.num_rows      ,   3_3	)
         self.assertEqual(dset.num_columns      ,   1	)

         lowercase    :			Any  =       ["train", "test[:33%]"]
         lowercase    :			int  =       [Split.TRAIN, ReadInstruction.from_spec("""test[:33%]"""	)]
         for instructions in [instructionsa, instructionsa]:
            lowercase    :			Any  =       [reader.read(snake_case      ,   snake_case      ,   snake_case	) for instr in instructions]
            lowercase    :			List[str]  =       (Dataset(**snake_case	) for dataset_kwargs in datasets_kwargs)
            self.assertEqual(train_dset["""filename"""][0]      ,   f'''{name}-train'''	)
            self.assertEqual(train_dset.num_rows      ,   1_0_0	)
            self.assertEqual(train_dset.num_columns      ,   1	)
            self.assertIsInstance(train_dset.split      ,   snake_case	)
            self.assertEqual(str(train_dset.split	)      ,   """train"""	)
            self.assertEqual(test_dset["""filename"""][0]      ,   f'''{name}-test'''	)
            self.assertEqual(test_dset.num_rows      ,   3_3	)
            self.assertEqual(test_dset.num_columns      ,   1	)
            self.assertIsInstance(test_dset.split      ,   snake_case	)
            self.assertEqual(str(test_dset.split	)      ,   """test[:33%]"""	)
            del train_dset, test_dset



   def    _UpperCAmelCase	(       self	) ->    Any:







      """simple docstring"""
      lowercase    :			Optional[int]  =       "my_name"
      lowercase    :			List[str]  =       SplitInfo(name="""train"""      ,   num_examples=1_0_0_0      ,   shard_lengths=[1_0_0] * 1_0	)
      lowercase    :			Union[str, Any]  =       [train_info]
      lowercase    :			List[str]  =       SplitDict()
      split_dict.add(snake_case	)
      lowercase    :			Optional[int]  =       DatasetInfo(splits=snake_case	)

      with tempfile.TemporaryDirectory() as tmp_dir:
         lowercase    :			Tuple  =       ReaderTest(snake_case      ,   snake_case	)

         lowercase    :			int  =       "train[:33%]"
         lowercase    :			int  =       Dataset(**reader.read(snake_case      ,   snake_case      ,   snake_case	)	)
         self.assertEqual(dset["""filename"""][0]      ,   f'''{name}-train-00000-of-00010'''	)
         self.assertEqual(dset["""filename"""][-1]      ,   f'''{name}-train-00003-of-00010'''	)
         self.assertEqual(dset.num_rows      ,   3_3_0	)
         self.assertEqual(dset.num_columns      ,   1	)





   def    _UpperCAmelCase	(       self	) ->    List[str]:







      """simple docstring"""
      lowercase    :			Optional[Any]  =       SplitInfo(name="""train"""      ,   num_examples=1_0_0	)
      lowercase    :			Dict  =       SplitInfo(name="""test"""      ,   num_examples=1_0_0	)
      lowercase    :			str  =       SplitDict()
      split_dict.add(snake_case	)
      split_dict.add(snake_case	)
      lowercase    :			Dict  =       DatasetInfo(splits=snake_case	)

      with tempfile.TemporaryDirectory() as tmp_dir:
         lowercase    :			str  =       ReaderTest(snake_case      ,   snake_case	)

         lowercase    :			int  =       [
             {"filename": os.path.join(snake_case      ,   """train"""	)},
             {"filename": os.path.join(snake_case      ,   """test"""	), "skip": 1_0, "take": 1_0},
         ]
         lowercase    :			Union[str, Any]  =       Dataset(**reader.read_files(snake_case      ,   original_instructions="""train+test[10:20]"""	)	)
         self.assertEqual(dset.num_rows      ,   1_1_0	)
         self.assertEqual(dset.num_columns      ,   1	)
         del dset




@pytest.mark.parametrize("""in_memory""" ,[False, True]    )
def 			__snake_case						(      __A ,__A ,__A    )		->				List[str]:
   lowercase    :			Dict  =       arrow_file
   with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
      lowercase    :			Any  =       ArrowReader.read_table(lowercase__ ,in_memory=lowercase__    )
   assert table.shape == dataset.data.shape
   assert set(table.column_names    ) == set(dataset.data.column_names    )
   assert dict(table.to_pydict()    ) == dict(dataset.data.to_pydict()    )  # to_pydict returns OrderedDict




@pytest.mark.parametrize("""in_memory""" ,[False, True]    )
def 			__snake_case						(      __A ,__A ,__A    )		->				str:
   lowercase    :			str  =       arrow_file
   lowercase    :			Any  =       ArrowReader("""""" ,lowercase__    )
   with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
      lowercase    :			List[Any]  =       reader.read_files([{"""filename""": filename}] ,in_memory=lowercase__    )
   assert dataset_kwargs.keys() == {"arrow_table", "info", "split"}
   lowercase    :			List[str]  =       dataset_kwargs["arrow_table"]
   assert table.shape == dataset.data.shape
   assert set(table.column_names    ) == set(dataset.data.column_names    )
   assert dict(table.to_pydict()    ) == dict(dataset.data.to_pydict()    )  # to_pydict returns OrderedDict




def 			__snake_case						(      )		->				List[Any]:
   assert ReadInstruction("""train""" ,to=10 ,unit="""abs"""    ).to_spec() == "train[:10]"
   assert ReadInstruction("""train""" ,from_=-80 ,to=10 ,unit="""%"""    ).to_spec() == "train[-80%:10%]"

   lowercase    :			Tuple  =       "train+test"
   assert ReadInstruction.from_spec(lowercase__    ).to_spec() == spec_train_test

   lowercase    :			str  =       "train[2:10]"
   assert ReadInstruction.from_spec(lowercase__    ).to_spec() == spec_train_abs

   lowercase    :			Optional[Any]  =       "train[15%:-20%]"
   assert ReadInstruction.from_spec(lowercase__    ).to_spec() == spec_train_pct

   lowercase    :			Union[str, Any]  =       "train[:10%](closest)"
   assert ReadInstruction.from_spec(lowercase__    ).to_spec() == "train[:10%]"

   lowercase    :			Dict  =       "train[:10%](pct1_dropremainder)"
   assert ReadInstruction.from_spec(lowercase__    ).to_spec() == spec_train_pct_rounding

   lowercase    :			Tuple  =       "train[:10%](pct1_dropremainder)+test[-10%:](pct1_dropremainder)"
   assert ReadInstruction.from_spec(lowercase__    ).to_spec() == spec_train_test_pct_rounding




def 			__snake_case						(      )		->				int:
   lowercase    :			List[str]  =       "dummy"
   lowercase    :			Optional[Any]  =       [SplitInfo(name="""train""" ,num_examples=100    )]
   lowercase    :			List[str]  =       "train[:33%]"
   lowercase    :			Optional[int]  =       "arrow"
   lowercase    :			Union[str, Any]  =       "prefix"

   lowercase    :			Tuple  =       make_file_instructions(lowercase__ ,lowercase__ ,lowercase__ ,lowercase__ ,lowercase__    )
   assert isinstance(lowercase__ ,lowercase__    )
   assert file_instructions.num_examples == 33
   assert file_instructions.file_instructions == [
       {"filename": os.path.join(lowercase__ ,F'''{name}-train.arrow'''    ), "skip": 0, "take": 33}
   ]

   lowercase    :			Tuple  =       [SplitInfo(name="""train""" ,num_examples=100 ,shard_lengths=[10] * 10    )]
   lowercase    :			List[str]  =       make_file_instructions(lowercase__ ,lowercase__ ,lowercase__ ,lowercase__ ,lowercase__    )
   assert isinstance(lowercase__ ,lowercase__    )
   assert file_instructions.num_examples == 33
   assert file_instructions.file_instructions == [
       {"filename": os.path.join(lowercase__ ,F'''{name}-train-00000-of-00010.arrow'''    ), "skip": 0, "take": -1},
       {"filename": os.path.join(lowercase__ ,F'''{name}-train-00001-of-00010.arrow'''    ), "skip": 0, "take": -1},
       {"filename": os.path.join(lowercase__ ,F'''{name}-train-00002-of-00010.arrow'''    ), "skip": 0, "take": -1},
       {"filename": os.path.join(lowercase__ ,F'''{name}-train-00003-of-00010.arrow'''    ), "skip": 0, "take": 3},
   ]







@pytest.mark.parametrize("""name, expected_exception""" ,[(None, TypeError), ("""""", ValueError)]    )
def 			__snake_case						(      __A ,__A    )		->				int:
   lowercase    :			Dict  =       [SplitInfo(name="""train""" ,num_examples=100    )]
   lowercase    :			Any  =       "train"
   lowercase    :			List[str]  =       "arrow"
   lowercase    :			Any  =       "prefix_path"
   with pytest.raises(lowercase__    ):
      lowercase    :			Optional[int]  =       make_file_instructions(lowercase__ ,lowercase__ ,lowercase__ ,lowercase__ ,lowercase__    )







"""simple docstring"""
from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy
class lowerCamelCase__  (    __UpperCamelCase				):
   __UpperCAmelCase        =					"""MarkupLMFeatureExtractor"""
   __UpperCAmelCase        =					("""MarkupLMTokenizer""", """MarkupLMTokenizerFast""")
   __UpperCAmelCase        =					True



   def __call__(       self      ,   snake_case=None      ,   snake_case=None      ,   snake_case=None      ,   snake_case=None      ,   snake_case=None      ,   snake_case = True      ,   snake_case = False      ,   snake_case = None      ,   snake_case = None      ,   snake_case = 0      ,   snake_case = None      ,   snake_case = None      ,   snake_case = None      ,   snake_case = False      ,   snake_case = False      ,   snake_case = False      ,   snake_case = False      ,   snake_case = True      ,   snake_case = None      ,   **snake_case      ,   ) ->    BatchEncoding:







      """simple docstring"""
      # first, create nodes and xpaths
      if self.parse_html:
         if html_strings is None:
            raise ValueError("""Make sure to pass HTML strings in case `parse_html` is set to `True`"""	)

         if nodes is not None or xpaths is not None or node_labels is not None:
            raise ValueError(
                """Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"""	)

         lowercase    :			List[Any]  =       self.feature_extractor(snake_case	)
         lowercase    :			Union[str, Any]  =       features["""nodes"""]
         lowercase    :			List[Any]  =       features["""xpaths"""]
      else:
         if html_strings is not None:
            raise ValueError("""You have passed HTML strings but `parse_html` is set to `False`."""	)
         if nodes is None or xpaths is None:
            raise ValueError("""Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"""	)

        # # second, apply the tokenizer
      if questions is not None and self.parse_html:
         if isinstance(snake_case      ,   snake_case	):
            lowercase    :			Dict  =       [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

      lowercase    :			Dict  =       self.tokenizer(
          text=questions if questions is not None else nodes      ,   text_pair=nodes if questions is not None else None      ,   xpaths=snake_case      ,   node_labels=snake_case      ,   add_special_tokens=snake_case      ,   padding=snake_case      ,   truncation=snake_case      ,   max_length=snake_case      ,   stride=snake_case      ,   pad_to_multiple_of=snake_case      ,   return_token_type_ids=snake_case      ,   return_attention_mask=snake_case      ,   return_overflowing_tokens=snake_case      ,   return_special_tokens_mask=snake_case      ,   return_offsets_mapping=snake_case      ,   return_length=snake_case      ,   verbose=snake_case      ,   return_tensors=snake_case      ,   **snake_case      ,   )

      return encoded_inputs



   def    _UpperCAmelCase	(       self      ,   *snake_case      ,   **snake_case	) ->    Optional[Any]:







      """simple docstring"""
      return self.tokenizer.batch_decode(*snake_case      ,   **snake_case	)



   def    _UpperCAmelCase	(       self      ,   *snake_case      ,   **snake_case	) ->    Tuple:







      """simple docstring"""
      return self.tokenizer.decode(*snake_case      ,   **snake_case	)





   @property
   def    _UpperCAmelCase	(       self	) ->    Dict:







      """simple docstring"""
      lowercase    :			Any  =       self.tokenizer.model_input_names
      return tokenizer_input_names


