






import json
import os
import tempfile

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features import ArrayaD
from utils import generate_examples, get_duration


UpperCAmelCase__            =	(30, 487)
UpperCAmelCase__            =	(36, 1024)
UpperCAmelCase__            =	(100, 100)
UpperCAmelCase__            =	100

UpperCAmelCase__            =	datasets.Features(
    {"text": ArrayaD(SHAPE_TEST_1, dtype="float32"), "image": ArrayaD(SHAPE_TEST_2, dtype="float32")}
)

UpperCAmelCase__     ,   UpperCAmelCase__            =	os.path.split(__file__)
UpperCAmelCase__            =	os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))
@get_duration
def 		_a					(   a				:Tuple			,							a				:int			,							a				:List[str]				)							->    List[Any]:
 with ArrowWriter(features=a			,							path=os.path.join(a			,							'''beta.arrow'''				)				) as writer:
  for key, record in dummy_data:
   a							=   my_features.encode_example(a				)
   writer.write(a				)
  a						,  a							=   writer.finalize()
@get_duration
def 		_a					(   a				:List[str]			,							a				:Optional[Any]				)							->    List[str]:
 a							=   datasets.Dataset.from_file(
     filename=os.path.join(a			,							'''beta.arrow'''				)			,							info=datasets.DatasetInfo(features=a				)				)
 for _ in dataset:
  pass
@get_duration
def 		_a					(   a				:str			,							a				:List[str]				)							->    Optional[Any]:
 a							=   datasets.Dataset.from_file(
     filename=os.path.join(a			,							'''beta.arrow'''				)			,							info=datasets.DatasetInfo(features=a				)				)
 dataset.set_format('''numpy'''				)
 for _ in dataset:
  pass
@get_duration
def 		_a					(   a				:Dict			,							a				:str				)							->    List[Any]:
 a							=   10
 a							=   datasets.Dataset.from_file(
     filename=os.path.join(a			,							'''beta.arrow'''				)			,							info=datasets.DatasetInfo(features=a				)				)
 for i in range(0			,							len(a				)			,							a				):
  a							=   dataset[i : i + batch_size]
@get_duration
def 		_a					(   a				:Optional[Any]			,							a				:Optional[int]				)							->    int:
 a							=   10
 a							=   datasets.Dataset.from_file(
     filename=os.path.join(a			,							'''beta.arrow'''				)			,							info=datasets.DatasetInfo(features=a				)				)
 dataset.set_format('''numpy'''				)
 for i in range(0			,							len(a				)			,							a				):
  a							=   dataset[i : i + batch_size]
@get_duration
def 		_a					(   a				:str			,							a				:List[Any]				)							->    int:
 a							=   datasets.Dataset.from_file(
     filename=os.path.join(a			,							'''beta.arrow'''				)			,							info=datasets.DatasetInfo(features=a				)				)
 for col in feats:
  a							=   dataset[col]
@get_duration
def 		_a					(   a				:Union[str, Any]			,							a				:Optional[Any]				)							->    str:
 a							=   datasets.Dataset.from_file(
     filename=os.path.join(a			,							'''beta.arrow'''				)			,							info=datasets.DatasetInfo(features=a				)				)
 dataset.set_format('''numpy'''				)
 for col in feats:
  a							=   dataset[col]
def 		_a					(   )							->    Tuple:
 a							=   {}
 a							=   (
     read_unformated,
     read_formatted_as_numpy,
     read_batch_unformated,
     read_batch_formatted_as_numpy,
     read_col_unformated,
     read_col_formatted_as_numpy,
 )
 with tempfile.TemporaryDirectory() as tmp_dir:
  a							=   datasets.Features({'''image''': ArrayaD(a			,							dtype='''float32'''				)}				)
  a							=   generate_examples(features=a			,							num_examples=a				)
  a							=   write(a			,							a			,							a				)
  for read_func in read_functions:
   a							=   read_func(a			,							a				)

 with tempfile.TemporaryDirectory() as tmp_dir:
  # don't use fixed length for fair comparison
  # feats = datasets.Features(
  #     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
  # )
  a							=   datasets.Features({'''image''': datasets.Sequence(datasets.Sequence(datasets.Value('''float32'''				)				)				)}				)
  a							=   generate_examples(
      features=a			,							num_examples=a			,							seq_shapes={'''image''': SPEED_TEST_SHAPE}				)
  a							=   write(a			,							a			,							a				)
  for read_func in read_functions:
   a							=   read_func(a			,							a				)

 with tempfile.TemporaryDirectory() as tmp_dir:
  # don't use fixed length for fair comparison
  # feats = datasets.Features(
  #     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
  # )
  a							=   datasets.Features({'''image''': datasets.Sequence(datasets.Value('''float32'''				)				)}				)
  a							=   generate_examples(
      features=a			,							num_examples=a			,							seq_shapes={'''image''': [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]}			,							)
  a							=   write(a			,							a			,							a				)
  for read_func in read_functions:
   a							=   read_func(a			,							a				)

 with open(a			,							'''wb'''				) as f:
  f.write(json.dumps(a				).encode('''utf-8'''				)				)


if __name__ == "__main__":  # useful to run the profiler
     benchmark_array_xd()










import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class 	lowercase_      (			lowercase     ):






      '''simple docstring'''




      __snake_case  						=						['''image_processor''', '''tokenizer''']
      __snake_case  						=						'''AutoImageProcessor'''
      __snake_case  						=						'''AutoTokenizer'''







      def __init__(   self   :		Any				,      __UpperCAmelCase   :		Union[str, Any]=None				,      __UpperCAmelCase   :		int=None				,      **__UpperCAmelCase   :		Union[str, Any]						)					->Dict:




       """simple docstring"""


       a							=   None
       if "feature_extractor" in kwargs:
        warnings.warn(
            '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
            ''' instead.'''				,      __UpperCAmelCase				,      )
        a							=   kwargs.pop('''feature_extractor'''						)

       a							=   image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
        raise ValueError('''You have to specify an image_processor.'''						)
       if tokenizer is None:
        raise ValueError('''You have to specify a tokenizer.'''						)

       super().__init__(__UpperCAmelCase				,      __UpperCAmelCase						)
       a							=   self.image_processor







      def __call__(   self   :		List[Any]				,      __UpperCAmelCase   :		Optional[Any]=None				,      __UpperCAmelCase   :		Optional[int]=None				,      __UpperCAmelCase   :		str=None				,      **__UpperCAmelCase   :		List[Any]						)					->Dict:




       """simple docstring"""


       if text is None and images is None:
        raise ValueError('''You have to specify either text or images. Both cannot be none.'''						)

       if text is not None:
        a							=   self.tokenizer(__UpperCAmelCase				,      return_tensors=__UpperCAmelCase				,      **__UpperCAmelCase						)

       if images is not None:
        a							=   self.image_processor(__UpperCAmelCase				,      return_tensors=__UpperCAmelCase				,      **__UpperCAmelCase						)

       if text is not None and images is not None:
        a							=   image_features.pixel_values
        return encoding
       elif text is not None:
        return encoding
       else:
        return BatchEncoding(data=dict(**__UpperCAmelCase						)				,      tensor_type=__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		List[Any]				,      *__UpperCAmelCase   :		Optional[Any]				,      **__UpperCAmelCase   :		List[str]						)					->Optional[Any]:




       """simple docstring"""


       return self.tokenizer.batch_decode(*__UpperCAmelCase				,      **__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		str				,      *__UpperCAmelCase   :		List[Any]				,      **__UpperCAmelCase   :		Optional[Any]						)					->Union[str, Any]:




       """simple docstring"""


       return self.tokenizer.decode(*__UpperCAmelCase				,      **__UpperCAmelCase						)







      @property
      def    __lowerCAmelCase      (   self   :		str						)					->str:




       """simple docstring"""


       a							=   self.tokenizer.model_input_names
       a							=   self.image_processor.model_input_names
       return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names						)						)







      @property
      def    __lowerCAmelCase      (   self   :		str						)					->Optional[Any]:




       """simple docstring"""


       warnings.warn(
           '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''				,      __UpperCAmelCase				,      )
       return self.image_processor_class







      @property
      def    __lowerCAmelCase      (   self   :		int						)					->Optional[Any]:




       """simple docstring"""


       warnings.warn(
           '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''				,      __UpperCAmelCase				,      )
       return self.image_processor


