





"""simple docstring"""




import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_	:		Optional[int]         =   logging.get_logger(__name__)

lowerCamelCase_	:		Optional[int]         =   {
    'facebook/hubert-base-ls960': 'https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}

class 					_UpperCAmelCase    (  UpperCAmelCase__						):



      '''simple docstring'''
      lowercase_ :					Optional[Any]             =  """hubert"""

      def __init__(	self   , snake_case_=3_2   , snake_case_=7_6_8   , snake_case_=1_2   , snake_case_=1_2   , snake_case_=3_0_7_2   , snake_case_="gelu"   , snake_case_=0.1   , snake_case_=0.1   , snake_case_=0.1   , snake_case_=True   , snake_case_=0.0   , snake_case_=0.1   , snake_case_=0.1   , snake_case_=0.02   , snake_case_=1E-5   , snake_case_="group"   , snake_case_="gelu"   , snake_case_=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2)   , snake_case_=(5, 2, 2, 2, 2, 2, 2)   , snake_case_=(1_0, 3, 3, 3, 3, 2, 2)   , snake_case_=False   , snake_case_=1_2_8   , snake_case_=1_6   , snake_case_=False   , snake_case_=True   , snake_case_=0.05   , snake_case_=1_0   , snake_case_=2   , snake_case_=0.0   , snake_case_=1_0   , snake_case_=0   , snake_case_="sum"   , snake_case_=False   , snake_case_=False   , snake_case_=2_5_6   , snake_case_=0   , snake_case_=1   , snake_case_=2   , **snake_case_   , ):

        """simple docstring"""

        super().__init__(**snake_case_   , pad_token_id=snake_case_   , bos_token_id=snake_case_   , eos_token_id=snake_case_    )
        A_    :			Tuple									=    hidden_size
        A_    :			Union[str, Any]									=    feat_extract_norm
        A_    :			Union[str, Any]									=    feat_extract_activation
        A_    :			Tuple									=    list(snake_case_    )
        A_    :			List[str]									=    list(snake_case_    )
        A_    :			Union[str, Any]									=    list(snake_case_    )
        A_    :			Optional[int]									=    conv_bias
        A_    :			Any									=    num_conv_pos_embeddings
        A_    :			int									=    num_conv_pos_embedding_groups
        A_    :			Tuple									=    len(self.conv_dim    )
        A_    :			Union[str, Any]									=    num_hidden_layers
        A_    :			List[Any]									=    intermediate_size
        A_    :			str									=    hidden_act
        A_    :			int									=    num_attention_heads
        A_    :			str									=    hidden_dropout
        A_    :			List[Any]									=    attention_dropout
        A_    :			Any									=    activation_dropout
        A_    :			Optional[int]									=    feat_proj_layer_norm
        A_    :			str									=    feat_proj_dropout
        A_    :			Any									=    final_dropout
        A_    :			Dict									=    layerdrop
        A_    :			Any									=    layer_norm_eps
        A_    :			List[str]									=    initializer_range
        A_    :			Tuple									=    vocab_size
        A_    :			List[str]									=    do_stable_layer_norm
        A_    :			Optional[Any]									=    use_weighted_layer_sum
        A_    :			List[str]									=    classifier_proj_size

        if (
            (len(self.conv_stride    ) != self.num_feat_extract_layers)
            or (len(self.conv_kernel    ) != self.num_feat_extract_layers)
            or (len(self.conv_dim    ) != self.num_feat_extract_layers)
        ):
          raise ValueError(
              'Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='
              ' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='
              F""" {len(self.conv_dim    )}`, `len(config.conv_stride) = {len(self.conv_stride    )}`,"""
              F""" `len(config.conv_kernel) = {len(self.conv_kernel    )}`."""    )

        # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
        A_    :			Union[str, Any]									=    apply_spec_augment
        A_    :			str									=    mask_time_prob
        A_    :			str									=    mask_time_length
        A_    :			Union[str, Any]									=    mask_time_min_masks
        A_    :			str									=    mask_feature_prob
        A_    :			int									=    mask_feature_length
        A_    :			List[str]									=    mask_feature_min_masks

        # ctc loss
        A_    :			Tuple									=    ctc_loss_reduction
        A_    :			int									=    ctc_zero_infinity

      @property
      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        return functools.reduce(operator.mul   , self.conv_stride   , 1    )






"""simple docstring"""




import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class 					_UpperCAmelCase    (  UpperCAmelCase__						):



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_ = None   , snake_case_ = None   , snake_case_ = None   , snake_case_ = False   , snake_case_ = False   , snake_case_ = None   , **snake_case_   , ):

        """simple docstring"""

        super().__init__(
            snake_case_   , split=snake_case_   , features=snake_case_   , cache_dir=snake_case_   , keep_in_memory=snake_case_   , streaming=snake_case_   , num_proc=snake_case_   , **snake_case_   , )
        A_    :			int									=    path_or_paths if isinstance(snake_case_   , snake_case_    ) else {self.split: path_or_paths}
        A_    :			List[str]									=    Csv(
            cache_dir=snake_case_   , data_files=snake_case_   , features=snake_case_   , **snake_case_   , )

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        if self.streaming:
          A_    :			List[str]									=    self.builder.as_streaming_dataset(split=self.split    )
        # Build regular (map-style) dataset
        else:
          A_    :			List[Any]									=    None
          A_    :			Optional[int]									=    None
          A_    :			Optional[int]									=    None
          A_    :			List[str]									=    None

          self.builder.download_and_prepare(
              download_config=snake_case_   , download_mode=snake_case_   , verification_mode=snake_case_   , base_path=snake_case_   , num_proc=self.num_proc   , )
          A_    :			List[Any]									=    self.builder.as_dataset(
              split=self.split   , verification_mode=snake_case_   , in_memory=self.keep_in_memory    )
        return dataset


class 					_UpperCAmelCase    :



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_   , snake_case_ = None   , snake_case_ = None   , **snake_case_   , ):

        """simple docstring"""

        if num_proc is not None and num_proc <= 0:
          raise ValueError(F"""num_proc {num_proc} must be an integer > 0."""    )

        A_    :			Optional[int]									=    dataset
        A_    :			List[str]									=    path_or_buf
        A_    :			Optional[int]									=    batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
        A_    :			Optional[int]									=    num_proc
        A_    :			Optional[int]									=    'utf-8'
        A_    :			int									=    to_csv_kwargs

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Union[str, Any]									=    self.to_csv_kwargs.pop('path_or_buf'   , snake_case_    )
        A_    :			str									=    self.to_csv_kwargs.pop('header'   , snake_case_    )
        A_    :			Optional[Any]									=    self.to_csv_kwargs.pop('index'   , snake_case_    )

        if isinstance(self.path_or_buf   , (str, bytes, os.PathLike)    ):
          with open(self.path_or_buf   , 'wb+'    ) as buffer:
            A_    :			Optional[Any]									=    self._write(file_obj=snake_case_   , header=snake_case_   , index=snake_case_   , **self.to_csv_kwargs    )
        else:
          A_    :			int									=    self._write(file_obj=self.path_or_buf   , header=snake_case_   , index=snake_case_   , **self.to_csv_kwargs    )
        return written

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_		,			A_		,			A_		,			A_    :			Union[str, Any]									=    args

        A_    :			Dict									=    query_table(
            table=self.dataset.data   , key=slice(snake_case_   , offset + self.batch_size    )   , indices=self.dataset._indices   , )
        A_    :			List[str]									=    batch.to_pandas().to_csv(
            path_or_buf=snake_case_   , header=header if (offset == 0) else False   , index=snake_case_   , **snake_case_    )
        return csv_str.encode(self.encoding    )

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_   , snake_case_   , **snake_case_    ):

        """simple docstring"""

        A_    :			Any									=    0

        if self.num_proc is None or self.num_proc == 1:
          for offset in logging.tqdm(
              range(0   , len(self.dataset    )   , self.batch_size    )   , unit='ba'   , disable=not logging.is_progress_bar_enabled()   , desc='Creating CSV from Arrow format'   , ):
            A_    :			Optional[int]									=    self._batch_csv((offset, header, index, to_csv_kwargs)    )
            written += file_obj.write(snake_case_    )

        else:
          A_		,			A_    :			Optional[int]									=    len(self.dataset    ), self.batch_size
          with multiprocessing.Pool(self.num_proc    ) as pool:
            for csv_str in logging.tqdm(
                pool.imap(
                    self._batch_csv   , [(offset, header, index, to_csv_kwargs) for offset in range(0   , snake_case_   , snake_case_    )]   , )   , total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size   , unit='ba'   , disable=not logging.is_progress_bar_enabled()   , desc='Creating CSV from Arrow format'   , ):
              written += file_obj.write(snake_case_    )

        return written