






"""simple docstring"""



import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


_lowerCAmelCase  :  Tuple   =			500_000

_lowerCAmelCase ,     _lowerCAmelCase  :  str   =			os.path.split(__file__)
_lowerCAmelCase  :  List[str]   =			os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))


@get_duration
def   SCREAMING_SNAKE_CASE__					(     snake_case					:   datasets.Dataset							)->		str:






     '''simple docstring'''



     UpperCAmelCase__	:						Optional[int]       =       dataset.select(range(0 ,				len(snake_case							) ,				2							)							)


@get_duration
def   SCREAMING_SNAKE_CASE__					(     snake_case					:   datasets.Dataset							)->		str:






     '''simple docstring'''



     UpperCAmelCase__	:						Optional[int]       =       dataset.sort("numbers"							)


@get_duration
def   SCREAMING_SNAKE_CASE__					(     snake_case					:   datasets.Dataset							)->		int:






     '''simple docstring'''



     UpperCAmelCase__	:						Union[str, Any]       =       dataset.shuffle()


@get_duration
def   SCREAMING_SNAKE_CASE__					(     snake_case					:   datasets.Dataset							)->		Optional[int]:






     '''simple docstring'''



     UpperCAmelCase__	:						Tuple       =       dataset.train_test_split(0.1							)


@get_duration
def   SCREAMING_SNAKE_CASE__					(     snake_case					:   datasets.Dataset ,				snake_case					:   Dict=10							)->		Dict:






     '''simple docstring'''



     for shard_id in range(snake_case							):
          UpperCAmelCase__	:						Dict       =       dataset.shard(snake_case ,				snake_case							)


def   SCREAMING_SNAKE_CASE__					(     )->		Optional[int]:






     '''simple docstring'''



     UpperCAmelCase__	:						Optional[int]       =       {"num examples": SPEED_TEST_N_EXAMPLES}
     UpperCAmelCase__	:						Tuple       =       (select, sort, shuffle, train_test_split, shard)
     with tempfile.TemporaryDirectory() as tmp_dir:
          print("generating dataset"							)
          UpperCAmelCase__	:						Any       =       datasets.Features({"text": datasets.Value("string"							), "numbers": datasets.Value("float32"							)}							)
          UpperCAmelCase__	:						List[str]       =       generate_example_dataset(
              os.path.join(snake_case ,				"dataset.arrow"							) ,				snake_case ,				num_examples=snake_case							)
          print("Functions"							)
          for func in functions:
               print(func.__name__							)
               UpperCAmelCase__	:						str       =       func(snake_case							)

     with open(snake_case ,				"wb"							) as f:
          f.write(json.dumps(snake_case							).encode("utf-8"							)							)


if __name__ == "__main__":  # useful to run the profiler
 benchmark_indices_mapping()










"""simple docstring"""



# Lint as: python3

import hashlib
from typing import Union


def   SCREAMING_SNAKE_CASE__					(     snake_case					:   Union[str, int, bytes]							)->		bytes:






     '''simple docstring'''



     if isinstance(snake_case ,				snake_case							):
          # Data already in bytes, returns as it as
          return hash_data
     elif isinstance(snake_case ,				snake_case							):
          # We keep the data as it as for it ot be later encoded to UTF-8
          # However replace `\\` with `/` for Windows compatibility
          UpperCAmelCase__	:						Union[str, Any]       =       hash_data.replace("\\" ,				"/"							)
     elif isinstance(snake_case ,				snake_case							):
          UpperCAmelCase__	:						int       =       str(snake_case							)
     else:
          # If data is not of the required type, raise error
          raise InvalidKeyError(snake_case							)

     return hash_data.encode("utf-8"							)




class  lowerCAmelCase__ (						__magic_name__      ):
   def __init__(		self			:  List[str]       , snake_case__			:  Union[str, Any]       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Tuple       =       "\nFAILURE TO GENERATE DATASET: Invalid key type detected"
        UpperCAmelCase__	:						str       =       f'\nFound Key {hash_data} of type {type(snake_case__       )}'
        UpperCAmelCase__	:						Union[str, Any]       =       "\nKeys should be either str, int or bytes type"
        super().__init__(f'{self.prefix}{self.err_msg}{self.suffix}'       )




class  lowerCAmelCase__ (						__magic_name__      ):
   def __init__(		self			:  List[str]       , snake_case__			:  Dict       , snake_case__			:  Optional[int]       , snake_case__			:  List[str]=""       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Union[str, Any]       =       key
        UpperCAmelCase__	:						Optional[Any]       =       duplicate_key_indices
        UpperCAmelCase__	:						str       =       fix_msg
        UpperCAmelCase__	:						Any       =       "Found multiple examples generated with the same key"
        if len(snake_case__       ) <= 2_0:
             UpperCAmelCase__	:						int       =       f'\nThe examples at index {", ".join(snake_case__       )} have the key {key}'
        else:
             UpperCAmelCase__	:						str       =       f'\nThe examples at index {", ".join(duplicate_key_indices[:2_0]       )}... ({len(snake_case__       ) - 2_0} more) have the key {key}'
        UpperCAmelCase__	:						Any       =       "\n" + fix_msg if fix_msg else ""
        super().__init__(f'{self.prefix}{self.err_msg}{self.suffix}'       )




class  lowerCAmelCase__ :
   def __init__(		self			:  Dict       , snake_case__			:  str       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Union[str, Any]       =       hashlib.mda(_as_bytes(snake_case__       )       )




   def 			__a       (		self			:  int       , snake_case__			:  Union[str, int, bytes]       ):


        '''simple docstring'''
        UpperCAmelCase__	:						int       =       self._split_mda.copy()
        UpperCAmelCase__	:						List[str]       =       _as_bytes(snake_case__       )
        mda.update(snake_case__       )
        # Convert to integer with hexadecimal conversion
        return int(mda.hexdigest()       , 1_6       )


