"""simple docstring"""






import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available

def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:Union[str, Any]		,			_SCREAMING_SNAKE_CASE	:Optional[int]		,			_SCREAMING_SNAKE_CASE	:Dict		,			_SCREAMING_SNAKE_CASE	:Union[str, Any]=None  )				-> Tuple:
						a_							:			Optional[int]     		=					None
						# Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
						if str(weight.dtype  ) == "torch.bfloat16":
												# Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
												a_							:			List[str]     		=					weight.view(torch.intaa  )
												a_							:			int     		=					"bfloat16"
						a_							:			int     		=					weight.cpu().numpy()
						a_							:			Tuple     		=					os.path.join(_SCREAMING_SNAKE_CASE		,			F'''{weight_name}.dat'''  )
						if index is not None:
												if dtype is None:
																		a_							:			List[str]     		=					str(array.dtype  )
												a_							:			Any     		=					{"dtype": dtype, "shape": list(array.shape  )}
						if array.ndim == 0:
												a_							:			Dict     		=					array[None]
						a_							:			str     		=					np.memmap(_SCREAMING_SNAKE_CASE		,			dtype=array.dtype		,			mode="w+"		,			shape=array.shape  )
						a_							:			List[Any]     		=					array[:]
						file_array.flush()
						return index

def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:Optional[int]		,			_SCREAMING_SNAKE_CASE	:Tuple  )				-> Tuple:
						a_							:			Optional[int]     		=					tuple(weight_info["shape"]  )
						if shape == ():
												# NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
												a_							:			int     		=					(1,)

						a_							:			List[str]     		=					weight_info["dtype"]
						if dtype == "bfloat16":
												# NumPy does not support bfloat16 so this was saved as a int16
												a_							:			Optional[int]     		=					"int16"

						a_							:			str     		=					np.memmap(_SCREAMING_SNAKE_CASE		,			dtype=_SCREAMING_SNAKE_CASE		,			shape=_SCREAMING_SNAKE_CASE		,			mode="r"  )

						if len(weight_info["shape"]  ) == 0:
												a_							:			List[str]     		=					weight[0]
						a_							:			Tuple     		=					torch.tensor(_SCREAMING_SNAKE_CASE  )
						if weight_info["dtype"] == "bfloat16":
												a_							:			Union[str, Any]     		=					weight.view(torch.bfloataa  )

						return weight

def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:str		,			_SCREAMING_SNAKE_CASE	:int  )				-> str:
						if index is None or len(_SCREAMING_SNAKE_CASE  ) == 0:
												# Nothing to save
												return

						a_							:			Union[str, Any]     		=					os.path.join(_SCREAMING_SNAKE_CASE		,			"index.json"  )
						if os.path.isfile(_SCREAMING_SNAKE_CASE  ):
												with open(_SCREAMING_SNAKE_CASE		,			"r"		,			encoding="utf-8"  ) as f:
																		a_							:			Any     		=					json.load(_SCREAMING_SNAKE_CASE  )
						else:
												a_							:			Tuple     		=					{}
						current_index.update(_SCREAMING_SNAKE_CASE  )

						with open(_SCREAMING_SNAKE_CASE		,			"w"		,			encoding="utf-8"  ) as f:
												json.dump(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE		,			indent=2  )

def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:Union[str, os.PathLike]		,			_SCREAMING_SNAKE_CASE	:Dict[str, torch.Tensor]  )				-> int:
						os.makedirs(_SCREAMING_SNAKE_CASE		,			exist_ok=_SCREAMING_SNAKE_CASE  )
						a_							:			Dict     		=					{}
						for name, parameter in state_dict.items():
												a_							:			Tuple     		=					offload_weight(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE		,			index=_SCREAMING_SNAKE_CASE  )

						# Update index
						save_offload_index(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )
class   UpperCAmelCase__				(    __lowerCamelCase			):


	"""simple docstring"""







	def __init__(			self       ,     _SCREAMING_SNAKE_CASE       ,     _SCREAMING_SNAKE_CASE )    ->	List[Any]:
							a_							:			List[Any]     		=					dataset
							a_							:			str     		=					prefix







	def __getitem__(			self       ,     _SCREAMING_SNAKE_CASE )    ->	str:
							return self.dataset[f'''{self.prefix}{key}''']







	def __iter__(			self )    ->	Dict:
							return iter([key for key in self.dataset if key.startswith(self.prefix )] )







	def __len__(			self )    ->	Dict:
							return len(self.dataset )




class   UpperCAmelCase__				(    __lowerCamelCase			):


	"""simple docstring"""







	def __init__(			self       ,     _SCREAMING_SNAKE_CASE = None       ,     _SCREAMING_SNAKE_CASE = None       ,     _SCREAMING_SNAKE_CASE = None       ,     _SCREAMING_SNAKE_CASE=None       ,     )    ->	Any:
							if state_dict is None and save_folder is None:
													raise ValueError("Need either a `state_dict` or a `save_folder` containing offloaded weights." )

							a_							:			Dict     		=					{} if state_dict is None else state_dict
							a_							:			Tuple     		=					save_folder
							if index is None and save_folder is not None:
													with open(os.path.join(_SCREAMING_SNAKE_CASE       ,     "index.json" ) ) as f:
																			a_							:			Optional[Any]     		=					json.load(_SCREAMING_SNAKE_CASE )
							a_							:			str     		=					{} if index is None else index
							a_							:			List[Any]     		=					list(self.state_dict.keys() )
							self.all_keys.extend([key for key in self.index if key not in self.all_keys] )
							a_							:			Dict     		=					device







	def __getitem__(			self       ,     _SCREAMING_SNAKE_CASE )    ->	Tuple:
							# State dict gets priority
							if key in self.state_dict:
													return self.state_dict[key]
							a_							:			Optional[Any]     		=					self.index[key]
							if weight_info.get("safetensors_file" ) is not None:
													if not is_safetensors_available():
																			raise ImportError("These offloaded weights require the use of safetensors: `pip install safetensors`." )

													from safetensors import safe_open

													a_							:			Union[str, Any]     		=					"cpu" if self.device is None else self.device
													with safe_open(weight_info["safetensors_file"]       ,     framework="pt"       ,     device=_SCREAMING_SNAKE_CASE ) as f:
																			a_							:			int     		=					f.get_tensor(weight_info.get("weight_name"       ,     _SCREAMING_SNAKE_CASE ) )

													if "dtype" in weight_info:
																			return tensor.to(getattr(_SCREAMING_SNAKE_CASE       ,     weight_info["dtype"] ) )
													else:
																			return tensor

							a_							:			Union[str, Any]     		=					os.path.join(self.save_folder       ,     f'''{key}.dat''' )
							return load_offloaded_weight(_SCREAMING_SNAKE_CASE       ,     _SCREAMING_SNAKE_CASE )







	def __iter__(			self )    ->	Dict:
							return iter(self.all_keys )







	def __len__(			self )    ->	Any:
							return len(self.all_keys )




def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:Dict[str, torch.Tensor]		,			_SCREAMING_SNAKE_CASE	:List[str]  )				-> Optional[int]:
						a_							:			Optional[Any]     		=					{}
						for module_name in submodule_names:
												# We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
												# submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
												result.update(
												    {
												        key: param
												        for key, param in state_dict.items()
												        if key == module_name or key.startswith(module_name + "."  )
												    }  )
						return result





"""simple docstring"""






import json
import os
import tempfile

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features import ArrayaD
from utils import generate_examples, get_duration


UpperCamelCase     					=     (30, 4_87)
UpperCamelCase     					=     (36, 10_24)
UpperCamelCase     					=     (1_00, 1_00)
UpperCamelCase     					=     1_00

UpperCamelCase     					=     datasets.Features(
    {'text': ArrayaD(SHAPE_TEST_1, dtype='float32'), 'image': ArrayaD(SHAPE_TEST_2, dtype='float32')}
)

UpperCamelCase					,      UpperCamelCase     					=     os.path.split(__file__)
UpperCamelCase     					=     os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:Tuple		,			_SCREAMING_SNAKE_CASE	:str		,			_SCREAMING_SNAKE_CASE	:Tuple  )				-> Any:
						with ArrowWriter(features=_SCREAMING_SNAKE_CASE		,			path=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )  ) as writer:
												for key, record in dummy_data:
																		a_							:			int     		=					my_features.encode_example(_SCREAMING_SNAKE_CASE  )
																		writer.write(_SCREAMING_SNAKE_CASE  )
												a_		,					a_							:			str     		=					writer.finalize()

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:List[str]		,			_SCREAMING_SNAKE_CASE	:Any  )				-> Tuple:
						a_							:			Dict     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						for _ in dataset:
												pass

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:str		,			_SCREAMING_SNAKE_CASE	:Dict  )				-> Optional[Any]:
						a_							:			List[str]     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						dataset.set_format("numpy"  )
						for _ in dataset:
												pass

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:List[Any]		,			_SCREAMING_SNAKE_CASE	:List[str]  )				-> List[str]:
						a_							:			Optional[Any]     		=					10
						a_							:			Optional[int]     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						for i in range(0		,			len(_SCREAMING_SNAKE_CASE  )		,			_SCREAMING_SNAKE_CASE  ):
												a_							:			List[str]     		=					dataset[i : i + batch_size]

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:Optional[int]		,			_SCREAMING_SNAKE_CASE	:List[str]  )				-> Optional[int]:
						a_							:			List[str]     		=					10
						a_							:			int     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						dataset.set_format("numpy"  )
						for i in range(0		,			len(_SCREAMING_SNAKE_CASE  )		,			_SCREAMING_SNAKE_CASE  ):
												a_							:			str     		=					dataset[i : i + batch_size]

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:List[str]		,			_SCREAMING_SNAKE_CASE	:Optional[Any]  )				-> Dict:
						a_							:			List[str]     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						for col in feats:
												a_							:			List[Any]     		=					dataset[col]

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:str		,			_SCREAMING_SNAKE_CASE	:Optional[Any]  )				-> Dict:
						a_							:			Any     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						dataset.set_format("numpy"  )
						for col in feats:
												a_							:			Any     		=					dataset[col]

def  lowerCAmelCase_      ()				-> Optional[Any]:
						a_							:			List[Any]     		=					{}
						a_							:			str     		=					(
						    read_unformated,
						    read_formatted_as_numpy,
						    read_batch_unformated,
						    read_batch_formatted_as_numpy,
						    read_col_unformated,
						    read_col_formatted_as_numpy,
						)
						with tempfile.TemporaryDirectory() as tmp_dir:
												a_							:			Optional[int]     		=					datasets.Features({"image": ArrayaD(_SCREAMING_SNAKE_CASE		,			dtype="float32"  )}  )
												a_							:			Optional[Any]     		=					generate_examples(features=_SCREAMING_SNAKE_CASE		,			num_examples=_SCREAMING_SNAKE_CASE  )
												a_							:			List[str]     		=					write(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )
												for read_func in read_functions:
																		a_							:			Optional[int]     		=					read_func(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )

						with tempfile.TemporaryDirectory() as tmp_dir:
												# don't use fixed length for fair comparison
												# feats = datasets.Features(
												#     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
												# )
												a_							:			Union[str, Any]     		=					datasets.Features({"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"  )  )  )}  )
												a_							:			List[str]     		=					generate_examples(
												    features=_SCREAMING_SNAKE_CASE		,			num_examples=_SCREAMING_SNAKE_CASE		,			seq_shapes={"image": SPEED_TEST_SHAPE}  )
												a_							:			str     		=					write(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )
												for read_func in read_functions:
																		a_							:			List[str]     		=					read_func(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )

						with tempfile.TemporaryDirectory() as tmp_dir:
												# don't use fixed length for fair comparison
												# feats = datasets.Features(
												#     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
												# )
												a_							:			int     		=					datasets.Features({"image": datasets.Sequence(datasets.Value("float32"  )  )}  )
												a_							:			str     		=					generate_examples(
												    features=_SCREAMING_SNAKE_CASE		,			num_examples=_SCREAMING_SNAKE_CASE		,			seq_shapes={"image": [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]}		,			)
												a_							:			List[Any]     		=					write(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )
												for read_func in read_functions:
																		a_							:			str     		=					read_func(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )

						with open(_SCREAMING_SNAKE_CASE		,			"wb"  ) as f:
												f.write(json.dumps(_SCREAMING_SNAKE_CASE  ).encode("utf-8"  )  )


if __name__ == "__main__":  # useful to run the profiler
				benchmark_array_xd()




