





'''simple docstring'''






import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


A__       :     str          =datasets.utils.logging.get_logger(__name__)




@dataclass
class 				UpperCAmelCase  (    datasets.BuilderConfig     ):
      _lowercase:    Optional[datasets.Features]		  =     None




class 				UpperCAmelCase  (    datasets.ArrowBasedBuilder     ):
      _lowercase:    Optional[int]		  =     ArrowConfig




      def        lowercase__					(    self    :				Optional[int]						)       ->   Dict:
            return datasets.DatasetInfo(features=self.config.features						)




      def        lowercase__					(    self    :				Optional[Any]    ,    __snake_case    :				Union[str, Any]						)       ->   Optional[Any]:
            if not self.config.data_files:
                  raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"						)
            _lowerCAmelCase							  =							dl_manager.download_and_extract(self.config.data_files						)
            if isinstance(__snake_case    ,    (str, list, tuple)						):
                  _lowerCAmelCase							  =							data_files
                  if isinstance(__snake_case    ,    __snake_case						):
                        _lowerCAmelCase							  =							[files]
                  # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                  _lowerCAmelCase							  =							[dl_manager.iter_files(__snake_case						) for file in files]
                  return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,    gen_kwargs={"""files""": files}						)]
            _lowerCAmelCase							  =							[]
            for split_name, files in data_files.items():
                  if isinstance(__snake_case    ,    __snake_case						):
                        _lowerCAmelCase							  =							[files]
                  # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                  _lowerCAmelCase							  =							[dl_manager.iter_files(__snake_case						) for file in files]
                  # Infer features is they are stoed in the arrow schema
                  if self.info.features is None:
                        for file in itertools.chain.from_iterable(__snake_case						):
                              with open(__snake_case    ,    """rb"""						) as f:
                                    _lowerCAmelCase							  =							datasets.Features.from_arrow_schema(pa.ipc.open_stream(__snake_case						).schema						)
                              break
                  splits.append(datasets.SplitGenerator(name=__snake_case    ,    gen_kwargs={"""files""": files}						)						)
            return splits




      def        lowercase__					(    self    :				Any    ,    __snake_case    :				pa.Table						)       ->   pa.Table:
            if self.info.features is not None:
                  # more expensive cast to support nested features with keys in a different order
                  # allows str <-> int/float or str to Audio for example
                  _lowerCAmelCase							  =							table_cast(__snake_case    ,    self.info.features.arrow_schema						)
            return pa_table




      def        lowercase__					(    self    :				Dict    ,    __snake_case    :				Optional[int]						)       ->   Union[str, Any]:
            for file_idx, file in enumerate(itertools.chain.from_iterable(__snake_case						)						):
                  with open(__snake_case    ,    """rb"""						) as f:
                        try:
                              for batch_idx, record_batch in enumerate(pa.ipc.open_stream(__snake_case						)						):
                                    _lowerCAmelCase							  =							pa.Table.from_batches([record_batch]						)
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield f"{file_idx}_{batch_idx}", self._cast_table(__snake_case						)
                        except ValueError as e:
                              logger.error(f"Failed to read file '{file}' with error {type(__snake_case						)}: {e}"						)
                              raise













'''simple docstring'''






import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
      from transformers.models.electra.modeling_flax_electra import (
          FlaxElectraForCausalLM,
          FlaxElectraForMaskedLM,
          FlaxElectraForMultipleChoice,
          FlaxElectraForPreTraining,
          FlaxElectraForQuestionAnswering,
          FlaxElectraForSequenceClassification,
          FlaxElectraForTokenClassification,
          FlaxElectraModel,
      )




class 				UpperCAmelCase  (    unittest.TestCase     ):




      def __init__(    self    :				str    ,    __snake_case    :				str    ,    __snake_case    :				str=13    ,    __snake_case    :				Optional[Any]=7    ,    __snake_case    :				str=True    ,    __snake_case    :				Any=True    ,    __snake_case    :				List[str]=True    ,    __snake_case    :				Any=True    ,    __snake_case    :				List[str]=99    ,    __snake_case    :				Dict=24    ,    __snake_case    :				Any=32    ,    __snake_case    :				List[str]=5    ,    __snake_case    :				Any=4    ,    __snake_case    :				Optional[Any]=37    ,    __snake_case    :				Union[str, Any]="gelu"    ,    __snake_case    :				Dict=0.1    ,    __snake_case    :				int=0.1    ,    __snake_case    :				Dict=5_12    ,    __snake_case    :				List[Any]=16    ,    __snake_case    :				Dict=2    ,    __snake_case    :				str=0.02    ,    __snake_case    :				Any=4    ,    )       ->   Union[str, Any]:
            _lowerCAmelCase							  =							parent
            _lowerCAmelCase							  =							batch_size
            _lowerCAmelCase							  =							seq_length
            _lowerCAmelCase							  =							is_training
            _lowerCAmelCase							  =							use_attention_mask
            _lowerCAmelCase							  =							use_token_type_ids
            _lowerCAmelCase							  =							use_labels
            _lowerCAmelCase							  =							vocab_size
            _lowerCAmelCase							  =							hidden_size
            _lowerCAmelCase							  =							embedding_size
            _lowerCAmelCase							  =							num_hidden_layers
            _lowerCAmelCase							  =							num_attention_heads
            _lowerCAmelCase							  =							intermediate_size
            _lowerCAmelCase							  =							hidden_act
            _lowerCAmelCase							  =							hidden_dropout_prob
            _lowerCAmelCase							  =							attention_probs_dropout_prob
            _lowerCAmelCase							  =							max_position_embeddings
            _lowerCAmelCase							  =							type_vocab_size
            _lowerCAmelCase							  =							type_sequence_label_size
            _lowerCAmelCase							  =							initializer_range
            _lowerCAmelCase							  =							num_choices




      def        lowercase__					(    self    :				Optional[Any]						)       ->   Optional[Any]:
            _lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.vocab_size						)

            _lowerCAmelCase							  =							None
            if self.use_attention_mask:
                  _lowerCAmelCase							  =							random_attention_mask([self.batch_size, self.seq_length]						)

            _lowerCAmelCase							  =							None
            if self.use_token_type_ids:
                  _lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.type_vocab_size						)

            _lowerCAmelCase							  =							ElectraConfig(
                vocab_size=self.vocab_size    ,    hidden_size=self.hidden_size    ,    embedding_size=self.embedding_size    ,    num_hidden_layers=self.num_hidden_layers    ,    num_attention_heads=self.num_attention_heads    ,    intermediate_size=self.intermediate_size    ,    hidden_act=self.hidden_act    ,    hidden_dropout_prob=self.hidden_dropout_prob    ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,    max_position_embeddings=self.max_position_embeddings    ,    type_vocab_size=self.type_vocab_size    ,    initializer_range=self.initializer_range    ,    )

            return config, input_ids, token_type_ids, attention_mask




      def        lowercase__					(    self    :				Any						)       ->   Optional[int]:
            _lowerCAmelCase							  =							self.prepare_config_and_inputs()
            _lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase							  =							config_and_inputs
            _lowerCAmelCase							  =							{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
            return config, inputs_dict




@require_flax
class 				UpperCAmelCase  (    snake_case_							,	unittest.TestCase     ):
      _lowercase:    str		  =     True

      _lowercase:    Optional[Any]		  =     (
          (
              FlaxElectraModel,
              FlaxElectraForCausalLM,
              FlaxElectraForMaskedLM,
              FlaxElectraForPreTraining,
              FlaxElectraForTokenClassification,
              FlaxElectraForQuestionAnswering,
              FlaxElectraForMultipleChoice,
              FlaxElectraForSequenceClassification,
          )
          if is_flax_available()
          else ()
      )




      def        lowercase__					(    self    :				Tuple						)       ->   Optional[int]:
            _lowerCAmelCase							  =							FlaxElectraModelTester(self						)




      @slow
      def        lowercase__					(    self    :				List[Any]						)       ->   str:
            for model_class_name in self.all_model_classes:
                  if model_class_name == FlaxElectraForMaskedLM:
                        _lowerCAmelCase							  =							model_class_name.from_pretrained("""google/electra-small-generator"""						)
                  else:
                        _lowerCAmelCase							  =							model_class_name.from_pretrained("""google/electra-small-discriminator"""						)
                  _lowerCAmelCase							  =							model(np.ones((1, 1)						)						)
                  self.assertIsNotNone(__snake_case						)






