'''simple docstring'''


import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class 	__SCREAMING_SNAKE_CASE		(			lowerCamelCase_			):







  '''simple docstring'''






  def __init__(					self ,						snake_case_ ,						snake_case_ = None ,						snake_case_ = None ,						snake_case_ = None ,						snake_case_ = False ,						snake_case_ = False ,						snake_case_ = None ,						**snake_case_ ,						):




       '''simple docstring'''

       super().__init__(
           snake_case_ ,						split=snake_case_ ,						features=snake_case_ ,						cache_dir=snake_case_ ,						keep_in_memory=snake_case_ ,						streaming=snake_case_ ,						num_proc=snake_case_ ,						**snake_case_ ,						)
       UpperCAmelCase_				:  Union[str, Any]  = path_or_paths if isinstance(snake_case_ ,						snake_case_		) else {self.split: path_or_paths}
       UpperCAmelCase_				:  List[Any]  = Csv(
           cache_dir=snake_case_ ,						data_files=snake_case_ ,						features=snake_case_ ,						**snake_case_ ,						)






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       if self.streaming:
            UpperCAmelCase_				:  int  = self.builder.as_streaming_dataset(split=self.split		)
       # Build regular (map-style) dataset
       else:
            UpperCAmelCase_				:  Optional[Any]  = None
            UpperCAmelCase_				:  int  = None
            UpperCAmelCase_				:  List[str]  = None
            UpperCAmelCase_				:  Tuple  = None

            self.builder.download_and_prepare(
                download_config=snake_case_ ,						download_mode=snake_case_ ,						verification_mode=snake_case_ ,						base_path=snake_case_ ,						num_proc=self.num_proc ,						)
            UpperCAmelCase_				:  Optional[Any]  = self.builder.as_dataset(
                split=self.split ,						verification_mode=snake_case_ ,						in_memory=self.keep_in_memory		)
       return dataset







class 	__SCREAMING_SNAKE_CASE		:







  '''simple docstring'''






  def __init__(					self ,						snake_case_ ,						snake_case_ ,						snake_case_ = None ,						snake_case_ = None ,						**snake_case_ ,						):




       '''simple docstring'''

       if num_proc is not None and num_proc <= 0:
            raise ValueError(F'''num_proc {num_proc} must be an integer > 0.'''		)

       UpperCAmelCase_				:  Optional[int]  = dataset
       UpperCAmelCase_				:  Optional[int]  = path_or_buf
       UpperCAmelCase_				:  Any  = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
       UpperCAmelCase_				:  int  = num_proc
       UpperCAmelCase_				:  Dict  = 'utf-8'
       UpperCAmelCase_				:  Tuple  = to_csv_kwargs


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[Any]  = self.to_csv_kwargs.pop('path_or_buf' ,						snake_case_		)
       UpperCAmelCase_				:  str  = self.to_csv_kwargs.pop('header' ,						snake_case_		)
       UpperCAmelCase_				:  Tuple  = self.to_csv_kwargs.pop('index' ,						snake_case_		)

       if isinstance(self.path_or_buf ,						(str, bytes, os.PathLike)		):
            with open(self.path_or_buf ,						'wb+'		) as buffer:
                 UpperCAmelCase_				:  Optional[int]  = self._write(file_obj=snake_case_ ,						header=snake_case_ ,						index=snake_case_ ,						**self.to_csv_kwargs		)
       else:
            UpperCAmelCase_				:  Union[str, Any]  = self._write(file_obj=self.path_or_buf ,						header=snake_case_ ,						index=snake_case_ ,						**self.to_csv_kwargs		)
       return written


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_  ,     UpperCAmelCase_  ,     UpperCAmelCase_  ,     UpperCAmelCase_				:  List[Any]  = args

       UpperCAmelCase_				:  List[Any]  = query_table(
           table=self.dataset.data ,						key=slice(snake_case_ ,						offset + self.batch_size		) ,						indices=self.dataset._indices ,						)
       UpperCAmelCase_				:  Dict  = batch.to_pandas().to_csv(
           path_or_buf=snake_case_ ,						header=header if (offset == 0) else False ,						index=snake_case_ ,						**snake_case_		)
       return csv_str.encode(self.encoding		)






  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ ,						snake_case_ ,						**snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_				:  int  = 0

       if self.num_proc is None or self.num_proc == 1:
            for offset in logging.tqdm(
                range(0 ,						len(self.dataset		) ,						self.batch_size		) ,						unit='ba' ,						disable=not logging.is_progress_bar_enabled() ,						desc='Creating CSV from Arrow format' ,						):
                 UpperCAmelCase_				:  Any  = self._batch_csv((offset, header, index, to_csv_kwargs)		)
                 written += file_obj.write(snake_case_		)

       else:
            UpperCAmelCase_  ,     UpperCAmelCase_				:  List[Any]  = len(self.dataset		), self.batch_size
            with multiprocessing.Pool(self.num_proc		) as pool:
                 for csv_str in logging.tqdm(
                     pool.imap(
                         self._batch_csv ,						[(offset, header, index, to_csv_kwargs) for offset in range(0 ,						snake_case_ ,						snake_case_		)] ,						) ,						total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size ,						unit='ba' ,						disable=not logging.is_progress_bar_enabled() ,						desc='Creating CSV from Arrow format' ,						):
                      written += file_obj.write(snake_case_		)

       return written








'''simple docstring'''







import os
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
						import torch

						from transformers import GPTaLMHeadModel


@require_torch
@require_sentencepiece
@require_tokenizers
class       lowerCAmelCase			(					unittest.TestCase      ):




		@slow
		@unittest.skip("Model is not available."   )
		def      lowercase							(					self   ):
								_SCREAMING_SNAKE_CASE					=			"nvidia/megatron-gpt2-345m/"
								if "MYDIR" in os.environ:
														_SCREAMING_SNAKE_CASE					=			os.path.join(os.environ["MYDIR"]       ,    UpperCamelCase   )
								_SCREAMING_SNAKE_CASE					=			GPTaLMHeadModel.from_pretrained(UpperCamelCase   )
								model.to(UpperCamelCase   )
								model.half()

								_SCREAMING_SNAKE_CASE					=			torch.tensor(
								    [[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]       ,    device=UpperCamelCase       ,    dtype=torch.long       ,    )

								with torch.no_grad():
														_SCREAMING_SNAKE_CASE					=			model(UpperCamelCase   ).logits

								_SCREAMING_SNAKE_CASE					=			torch.Size((1, 9, 50_257)   )
								self.assertEqual(output.shape       ,    UpperCamelCase   )

								_SCREAMING_SNAKE_CASE					=			torch.tensor(
								    [
								        4.94_14,
								        -0.29_20,
								        -1.21_48,
								        -4.02_73,
								        -0.51_61,
								        -5.21_09,
								        -1.24_12,
								        -1.83_01,
								        -1.77_34,
								        -4.71_48,
								        -0.23_17,
								        -1.08_11,
								        -2.17_77,
								        0.41_41,
								        -3.79_69,
								        -4.05_86,
								        -2.53_32,
								        -3.38_09,
								        4.38_67,
								    ]       ,    device=UpperCamelCase       ,    dtype=torch.half       ,    )

								for i in range(19   ):
														_SCREAMING_SNAKE_CASE,				_SCREAMING_SNAKE_CASE					=			8 * i // 17, 2_792 * i  # along the diagonal
														_SCREAMING_SNAKE_CASE,				_SCREAMING_SNAKE_CASE					=			output[0, r, c], expected_diag[i]
														_SCREAMING_SNAKE_CASE					=			F'row={r} col={c} computed={computed} expected={expected}'
														self.assertAlmostEqual(UpperCamelCase       ,    UpperCamelCase       ,    delta=1e-4       ,    msg=UpperCamelCase   )