
'''simple docstring'''





import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A							     = logging.get_logger(__name__)

A							     = {
    "facebook/hubert-base-ls960": "https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json",
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}



class 			lowerCAmelCase__ (							UpperCamelCase							):



  '''simple docstring'''






  _UpperCamelCase							=			'hubert'

  def __init__(self	:			Optional[Any]						,			lowerCAmelCase_	:			Tuple=32						,			lowerCAmelCase_	:			Optional[Any]=7_68						,			lowerCAmelCase_	:			Dict=12						,			lowerCAmelCase_	:			str=12						,			lowerCAmelCase_	:			int=30_72						,			lowerCAmelCase_	:			Dict="gelu"						,			lowerCAmelCase_	:			str=0.1						,			lowerCAmelCase_	:			Union[str, Any]=0.1						,			lowerCAmelCase_	:			Union[str, Any]=0.1						,			lowerCAmelCase_	:			Tuple=True						,			lowerCAmelCase_	:			Union[str, Any]=0.0						,			lowerCAmelCase_	:			Any=0.1						,			lowerCAmelCase_	:			Any=0.1						,			lowerCAmelCase_	:			Dict=0.02						,			lowerCAmelCase_	:			int=1E-5						,			lowerCAmelCase_	:			int="group"						,			lowerCAmelCase_	:			Optional[Any]="gelu"						,			lowerCAmelCase_	:			Union[str, Any]=(5_12, 5_12, 5_12, 5_12, 5_12, 5_12, 5_12)						,			lowerCAmelCase_	:			Tuple=(5, 2, 2, 2, 2, 2, 2)						,			lowerCAmelCase_	:			Tuple=(10, 3, 3, 3, 3, 2, 2)						,			lowerCAmelCase_	:			int=False						,			lowerCAmelCase_	:			str=1_28						,			lowerCAmelCase_	:			Optional[Any]=16						,			lowerCAmelCase_	:			List[Any]=False						,			lowerCAmelCase_	:			List[str]=True						,			lowerCAmelCase_	:			List[Any]=0.05						,			lowerCAmelCase_	:			Optional[int]=10						,			lowerCAmelCase_	:			Any=2						,			lowerCAmelCase_	:			List[Any]=0.0						,			lowerCAmelCase_	:			Optional[int]=10						,			lowerCAmelCase_	:			Union[str, Any]=0						,			lowerCAmelCase_	:			Optional[Any]="sum"						,			lowerCAmelCase_	:			Any=False						,			lowerCAmelCase_	:			List[Any]=False						,			lowerCAmelCase_	:			Union[str, Any]=2_56						,			lowerCAmelCase_	:			int=0						,			lowerCAmelCase_	:			Tuple=1						,			lowerCAmelCase_	:			List[str]=2						,			**lowerCAmelCase_	:			List[Any]						,			) ->							Any:
        super().__init__(**lowerCAmelCase_						,			pad_token_id=lowerCAmelCase_						,			bos_token_id=lowerCAmelCase_						,			eos_token_id=lowerCAmelCase_     )
        snake_case_    							=				hidden_size
        snake_case_    							=				feat_extract_norm
        snake_case_    							=				feat_extract_activation
        snake_case_    							=				list(lowerCAmelCase_     )
        snake_case_    							=				list(lowerCAmelCase_     )
        snake_case_    							=				list(lowerCAmelCase_     )
        snake_case_    							=				conv_bias
        snake_case_    							=				num_conv_pos_embeddings
        snake_case_    							=				num_conv_pos_embedding_groups
        snake_case_    							=				len(self.conv_dim     )
        snake_case_    							=				num_hidden_layers
        snake_case_    							=				intermediate_size
        snake_case_    							=				hidden_act
        snake_case_    							=				num_attention_heads
        snake_case_    							=				hidden_dropout
        snake_case_    							=				attention_dropout
        snake_case_    							=				activation_dropout
        snake_case_    							=				feat_proj_layer_norm
        snake_case_    							=				feat_proj_dropout
        snake_case_    							=				final_dropout
        snake_case_    							=				layerdrop
        snake_case_    							=				layer_norm_eps
        snake_case_    							=				initializer_range
        snake_case_    							=				vocab_size
        snake_case_    							=				do_stable_layer_norm
        snake_case_    							=				use_weighted_layer_sum
        snake_case_    							=				classifier_proj_size

        if (
            (len(self.conv_stride     ) != self.num_feat_extract_layers)
            or (len(self.conv_kernel     ) != self.num_feat_extract_layers)
            or (len(self.conv_dim     ) != self.num_feat_extract_layers)
        ):
              raise ValueError(
                  "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
                  " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
                  f''' {len(self.conv_dim     )}`, `len(config.conv_stride) = {len(self.conv_stride     )}`,'''
                  f''' `len(config.conv_kernel) = {len(self.conv_kernel     )}`.'''     )

        # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
        snake_case_    							=				apply_spec_augment
        snake_case_    							=				mask_time_prob
        snake_case_    							=				mask_time_length
        snake_case_    							=				mask_time_min_masks
        snake_case_    							=				mask_feature_prob
        snake_case_    							=				mask_feature_length
        snake_case_    							=				mask_feature_min_masks

        # ctc loss
        snake_case_    							=				ctc_loss_reduction
        snake_case_    							=				ctc_zero_infinity


  @property
  def 		_lowerCamelCase						(self	:			List[Any]     ) ->							List[Any]:
        return functools.reduce(operator.mul						,			self.conv_stride						,			1     )



import unittest
from functools import partial

import torch

from accelerate import Accelerator, debug_launcher
from accelerate.state import AcceleratorState, GradientState
from accelerate.test_utils import require_cpu, require_huggingface_suite
from accelerate.utils import GradientAccumulationPlugin

def 					SCREAMING_SNAKE_CASE  (   __UpperCamelCase			:    Any=2				,    __UpperCamelCase			:    str=True				,    __UpperCamelCase			:    Optional[Any]=False							)			->							Union[str, Any]:

				"""simple docstring"""







				A__	:						List[str]								=				Accelerator(step_scheduler_with_optimizer=__UpperCamelCase				,    split_batches=__UpperCamelCase							)
				A__	:						str								=				torch.nn.Linear(2				,    4							)
				A__	:						Any								=				torch.optim.AdamW(model.parameters()				,    lr=1.0							)
				A__	:						Tuple								=				torch.optim.lr_scheduler.OneCycleLR(__UpperCamelCase				,    max_lr=0.0_1				,    steps_per_epoch=2				,    epochs=1							)
				A__							,				A__							,				A__	:						str								=				accelerator.prepare(__UpperCamelCase				,    __UpperCamelCase				,    __UpperCamelCase							)

				# Optimizer has stepped
				scheduler.step()
				if step_scheduler_with_optimizer or (num_processes == 1):
								assert (
								    scheduler.scheduler.last_epoch == num_processes
								), F"Last Epoch ({scheduler.scheduler.last_epoch}) != Num Processes ({num_processes})"
				else:
								assert (
								    scheduler.scheduler.last_epoch != num_processes
								), F"Last Epoch ({scheduler.scheduler.last_epoch}) == Num Processes ({num_processes})"

def 					SCREAMING_SNAKE_CASE  (   __UpperCamelCase			:    Optional[Any]=2				,    __UpperCamelCase			:    Tuple=True				,    __UpperCamelCase			:    List[Any]=False							)			->							str:

				"""simple docstring"""







				A__	:						Union[str, Any]								=				Accelerator(step_scheduler_with_optimizer=__UpperCamelCase				,    split_batches=__UpperCamelCase							)
				A__	:						List[str]								=				torch.nn.Linear(2				,    4							)
				A__	:						int								=				torch.optim.AdamW(model.parameters()				,    lr=1.0							)
				A__	:						List[Any]								=				torch.optim.lr_scheduler.LambdaLR(__UpperCamelCase				,    lr_lambda=lambda __UpperCamelCase							: 1 - n / 10							)
				A__							,				A__							,				A__	:						int								=				accelerator.prepare(__UpperCamelCase				,    __UpperCamelCase				,    __UpperCamelCase							)

				# Optimizer has stepped
				A__	:						List[Any]								=				False
				scheduler.step()
				A__	:						str								=				1 - (num_processes if (step_scheduler_with_optimizer and not split_batches) else 1) / 10
				assert (
				    scheduler.get_last_lr()[0] == expected_lr
				), F"Wrong lr found at first step, expected {expected_lr}, got {scheduler.get_last_lr()[0]}"

				# Optimizer has not stepped
				A__	:						str								=				True
				scheduler.step()
				if not step_scheduler_with_optimizer:
								A__	:						Dict								=				1 - 2 / 10
				assert (
				    scheduler.get_last_lr()[0] == expected_lr
				), F"Wrong lr found at second step, expected {expected_lr}, got {scheduler.get_last_lr()[0]}"

def 					SCREAMING_SNAKE_CASE  (   __UpperCamelCase			:    int = 2							)			->							List[Any]:

				"""simple docstring"""







				from transformers import get_linear_schedule_with_warmup

				A__	:						str								=				[1, 2, 4]
				for num_steps in steps:
								A__	:						Any								=				GradientAccumulationPlugin(num_steps=__UpperCamelCase				,    adjust_scheduler=num_steps > 1							)
								A__	:						Tuple								=				Accelerator(gradient_accumulation_plugin=__UpperCamelCase							)
								A__	:						Optional[int]								=				torch.nn.Linear(2				,    4							)
								A__	:						Optional[int]								=				torch.optim.AdamW(model.parameters()				,    lr=1_0.0							)
								A__	:						Optional[int]								=				get_linear_schedule_with_warmup(optimizer=__UpperCamelCase				,    num_warmup_steps=0				,    num_training_steps=20							)

								A__							,				A__							,				A__	:						List[Any]								=				accelerator.prepare(__UpperCamelCase				,    __UpperCamelCase				,    __UpperCamelCase							)

								for i in range(10 * num_steps							):
												with accelerator.accumulate(__UpperCamelCase							):
																optimizer.step()
																scheduler.step()

												if i == (10 * num_steps - 2):
																assert (
																    scheduler.get_last_lr()[0] != 0
																), F"Wrong lr found at second-to-last step, expected non-zero, got {scheduler.get_last_lr()[0]}. num_steps: {num_steps}"
								assert (
								    scheduler.get_last_lr()[0] == 0
								), F"Wrong lr found at last step, expected 0, got {scheduler.get_last_lr()[0]}"
								GradientState._reset_state()







@require_cpu
class 			UpperCamelCase__		(     unittest.TestCase ):







	'''simple docstring'''





	def 	__snake_case					(      self		):
					debug_launcher(partial(UpperCamelCase__					,							num_processes=1		)					,							num_processes=1		)
					debug_launcher(partial(UpperCamelCase__					,							num_processes=1					,							split_batches=UpperCamelCase__		)					,							num_processes=1		)





	def 	__snake_case					(      self		):
					debug_launcher(partial(UpperCamelCase__					,							num_processes=1		)					,							num_processes=1		)
					debug_launcher(partial(UpperCamelCase__					,							num_processes=1					,							split_batches=UpperCamelCase__		)					,							num_processes=1		)





	def 	__snake_case					(      self		):
					debug_launcher(partial(UpperCamelCase__					,							num_processes=1					,							step_scheduler_with_optimizer=UpperCamelCase__		)					,							num_processes=1		)





	def 	__snake_case					(      self		):
					debug_launcher(partial(UpperCamelCase__					,							num_processes=1					,							step_scheduler_with_optimizer=UpperCamelCase__		)					,							num_processes=1		)





	def 	__snake_case					(      self		):
					AcceleratorState._reset_state(UpperCamelCase__		)
					debug_launcher(UpperCamelCase__		)
					debug_launcher(partial(UpperCamelCase__					,							num_processes=1					,							split_batches=UpperCamelCase__		)					,							num_processes=1		)





	def 	__snake_case					(      self		):
					AcceleratorState._reset_state(UpperCamelCase__		)
					debug_launcher(UpperCamelCase__		)
					debug_launcher(partial(UpperCamelCase__					,							num_processes=1					,							split_batches=UpperCamelCase__		)					,							num_processes=1		)





	def 	__snake_case					(      self		):
					AcceleratorState._reset_state(UpperCamelCase__		)
					debug_launcher(partial(UpperCamelCase__					,							step_scheduler_with_optimizer=UpperCamelCase__		)		)





	def 	__snake_case					(      self		):
					AcceleratorState._reset_state(UpperCamelCase__		)
					debug_launcher(partial(UpperCamelCase__					,							step_scheduler_with_optimizer=UpperCamelCase__		)		)





	@require_huggingface_suite
	def 	__snake_case					(      self		):
					AcceleratorState._reset_state(UpperCamelCase__		)
					debug_launcher(partial(UpperCamelCase__					,							num_processes=1		)		)
					debug_launcher(UpperCamelCase__		)