






import argparse
import json
import os

import fairseq
import torch
from fairseq.data import Dictionary

# Register SEW's fairseq modules
from sew_asapp import tasks  # noqa: F401

from transformers import (
    SEWDConfig,
    SEWDForCTC,
    SEWDModel,
    WavaVecaCTCTokenizer,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
    logging,
)


logging.set_verbosity_info()
__magic_name__:  List[Any]  	=			logging.get_logger(__name__)

__magic_name__:  Tuple  	=			{
    "post_extract_proj": "feature_projection",
    "encoder.pos_conv.0": "encoder.pos_conv_embed.conv",
    "attention.self.query_proj": "encoder.encoder.layer.*.attention.self.query_proj",
    "attention.self.key_proj": "encoder.encoder.layer.*.attention.self.key_proj",
    "attention.self.value_proj": "encoder.encoder.layer.*.attention.self.value_proj",
    "attention.output.dense": "encoder.encoder.layer.*.attention.output.dense",
    "attention.output.LayerNorm": "encoder.encoder.layer.*.attention.output.LayerNorm",
    "intermediate.dense": "encoder.encoder.layer.*.intermediate.dense",
    "output.dense": "encoder.encoder.layer.*.output.dense",
    "output.LayerNorm": "encoder.encoder.layer.*.output.LayerNorm",
    "encoder.encoder.rel_embeddings": "encoder.encoder.rel_embeddings",
    "encoder.encoder.LayerNorm": "encoder.encoder.LayerNorm",
    "encoder.upsample.0": "encoder.upsample.projection",
    "encoder.layer_norm": "encoder.layer_norm",
    "w2v_model.layer_norm": "layer_norm",
    "w2v_encoder.proj": "lm_head",
    "mask_emb": "masked_spec_embed",
}

def 					UpperCamelCase		( _A,					_A,					_A,					_A,					_A				):






						"""simple docstring"""







						for attribute in key.split("""."""				):
												__magic_name__    :     Tuple     		=      getattr(__snake_case,					__snake_case				)

						if weight_type is not None:
												__magic_name__    :     str     		=      getattr(__snake_case,					__snake_case				).shape
						else:
												__magic_name__    :     Any     		=      hf_pointer.shape

						assert hf_shape == value.shape, (
						    f'Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'
						    f' {value.shape} for {full_name}'
						)

						if weight_type == "weight":
												__magic_name__    :     Optional[Any]     		=      value
						elif weight_type == "weight_g":
												__magic_name__    :     Optional[int]     		=      value
						elif weight_type == "weight_v":
												__magic_name__    :     Optional[int]     		=      value
						elif weight_type == "bias":
												__magic_name__    :     Any     		=      value
						else:
												__magic_name__    :     List[Any]     		=      value

						logger.info(f'{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}.'				)

def 					UpperCamelCase		( _A,					_A,					_A				):






						"""simple docstring"""







						__magic_name__    :     List[str]     		=      []
						__magic_name__    :     List[Any]     		=      fairseq_model.state_dict()

						__magic_name__    :     Union[str, Any]     		=      hf_model.sew_d.feature_extractor if is_finetuned else hf_model.feature_extractor

						for name, value in fairseq_dict.items():
												__magic_name__    :     Optional[int]     		=      False
												if "conv_layers" in name:
																		load_conv_layer(
																		    __snake_case,					__snake_case,					__snake_case,					__snake_case,					hf_model.config.feat_extract_norm == """group""",					)
																		__magic_name__    :     str     		=      True
												else:
																		for key, mapped_key in MAPPING.items():
																								__magic_name__    :     Optional[int]     		=      """sew_d.""" + mapped_key if (is_finetuned and mapped_key != """lm_head""") else mapped_key

																								if key in name or key.split("""w2v_model."""				)[-1] == name.split("""."""				)[0]:
																														__magic_name__    :     Optional[int]     		=      True
																														if "*" in mapped_key:
																																				__magic_name__    :     str     		=      name.split(__snake_case				)[0].split("""."""				)[-2]
																																				if not layer_index.isnumeric():
																																										continue
																																				__magic_name__    :     Dict     		=      mapped_key.replace("""*""",					__snake_case				)
																														if "weight_g" in name:
																																				__magic_name__    :     Optional[int]     		=      """weight_g"""
																														elif "weight_v" in name:
																																				__magic_name__    :     Tuple     		=      """weight_v"""
																														elif "weight" in name:
																																				__magic_name__    :     Union[str, Any]     		=      """weight"""
																														elif "bias" in name:
																																				__magic_name__    :     Union[str, Any]     		=      """bias"""
																														else:
																																				__magic_name__    :     Optional[Any]     		=      None
																														set_recursively(__snake_case,					__snake_case,					__snake_case,					__snake_case,					__snake_case				)
																								continue
												if not is_used:
																		unused_weights.append(__snake_case				)

						logger.warning(f'Unused weights: {unused_weights}'				)

def 					UpperCamelCase		( _A,					_A,					_A,					_A,					_A				):






						"""simple docstring"""







						__magic_name__    :     Any     		=      full_name.split("""conv_layers."""				)[-1]
						__magic_name__    :     Optional[int]     		=      name.split("""."""				)
						__magic_name__    :     int     		=      int(items[0]				)
						__magic_name__    :     Union[str, Any]     		=      int(items[1]				)

						if type_id == 0:
												if "bias" in name:
																		assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
																		    f'{full_name} has size {value.shape}, but'
																		    f' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'
																		)
																		__magic_name__    :     Any     		=      value
																		logger.info(f'Feat extract conv layer {layer_id} was initialized from {full_name}.'				)
												elif "weight" in name:
																		assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
																		    f'{full_name} has size {value.shape}, but'
																		    f' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'
																		)
																		__magic_name__    :     List[str]     		=      value
																		logger.info(f'Feat extract conv layer {layer_id} was initialized from {full_name}.'				)
						elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
												if "bias" in name:
																		assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
																		    f'{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was'
																		    " found."
																		)
																		__magic_name__    :     Optional[int]     		=      value
																		logger.info(f'Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'				)
												elif "weight" in name:
																		assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
																		    f'{full_name} has size {value.shape}, but'
																		    f' {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.'
																		)
																		__magic_name__    :     List[Any]     		=      value
																		logger.info(f'Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'				)
						else:
												unused_weights.append(__snake_case				)

def 					UpperCamelCase		( _A,					_A				):






						"""simple docstring"""







						__magic_name__    :     Union[str, Any]     		=      SEWDConfig()
						if is_finetuned:
												__magic_name__    :     int     		=      model.wav_encoder.wav_model.cfg
						else:
												__magic_name__    :     Tuple     		=      model.cfg

						__magic_name__    :     Optional[Any]     		=      fs_config.conv_bias
						__magic_name__    :     Union[str, Any]     		=      eval(fs_config.conv_feature_layers				)
						__magic_name__    :     Any     		=      [x[0] for x in conv_layers]
						__magic_name__    :     Dict     		=      [x[1] for x in conv_layers]
						__magic_name__    :     Optional[int]     		=      [x[2] for x in conv_layers]
						__magic_name__    :     Optional[Any]     		=      """gelu"""
						__magic_name__    :     str     		=      """layer""" if fs_config.extractor_mode == """layer_norm""" else """group"""
						__magic_name__    :     List[Any]     		=      0.0
						__magic_name__    :     str     		=      fs_config.activation_fn.name
						__magic_name__    :     Any     		=      fs_config.encoder_embed_dim
						__magic_name__    :     List[str]     		=      0.02
						__magic_name__    :     str     		=      fs_config.encoder_ffn_embed_dim
						__magic_name__    :     Dict     		=      1e-5
						__magic_name__    :     Optional[Any]     		=      fs_config.encoder_layerdrop
						__magic_name__    :     Any     		=      fs_config.encoder_attention_heads
						__magic_name__    :     Union[str, Any]     		=      fs_config.conv_pos_groups
						__magic_name__    :     str     		=      fs_config.conv_pos
						__magic_name__    :     List[str]     		=      len(__snake_case				)
						__magic_name__    :     List[str]     		=      fs_config.encoder_layers
						__magic_name__    :     Optional[Any]     		=      fs_config.squeeze_factor
						# DeBERTa-specific parameters:
						__magic_name__    :     str     		=      fs_config.max_position_embeddings
						__magic_name__    :     Optional[int]     		=      fs_config.position_buckets
						__magic_name__    :     Any     		=      fs_config.share_att_key
						__magic_name__    :     int     		=      fs_config.relative_attention
						__magic_name__    :     List[str]     		=      fs_config.position_biased_input
						__magic_name__    :     List[str]     		=      tuple(fs_config.pos_att_type.split("""|"""				)				)
						__magic_name__    :     Dict     		=      fs_config.norm_rel_ebd

						# take care of any params that are overridden by the Wav2VecCtc model
						if is_finetuned:
												__magic_name__    :     Any     		=      model.cfg
												__magic_name__    :     Any     		=      fs_config.final_dropout
												__magic_name__    :     Optional[Any]     		=      fs_config.layerdrop
						__magic_name__    :     Any     		=      fs_config.activation_dropout
						__magic_name__    :     Any     		=      fs_config.mask_prob > 0 or fs_config.mask_channel_prob > 0
						__magic_name__    :     Optional[int]     		=      fs_config.attention_dropout
						__magic_name__    :     Union[str, Any]     		=      fs_config.dropout_input
						__magic_name__    :     List[Any]     		=      fs_config.dropout
						__magic_name__    :     Union[str, Any]     		=      fs_config.mask_channel_length
						__magic_name__    :     int     		=      fs_config.mask_channel_prob
						__magic_name__    :     List[str]     		=      fs_config.mask_length
						__magic_name__    :     Optional[int]     		=      fs_config.mask_prob

						__magic_name__    :     Union[str, Any]     		=      """Wav2Vec2FeatureExtractor"""
						__magic_name__    :     List[Any]     		=      """Wav2Vec2CTCTokenizer"""

						return config

@torch.no_grad()
def 					UpperCamelCase		( _A,					_A,					_A=None,					_A=None,					_A=True				):






						"""simple docstring"""







						if is_finetuned:
												__magic_name__		,__magic_name__		,__magic_name__    :     Union[str, Any]     		=      fairseq.checkpoint_utils.load_model_ensemble_and_task(
												    [checkpoint_path],					arg_overrides={"""data""": """/""".join(dict_path.split("""/"""				)[:-1]				)}				)
						else:
												__magic_name__		,__magic_name__		,__magic_name__    :     Tuple     		=      fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path]				)

						if config_path is not None:
												__magic_name__    :     Optional[Any]     		=      SEWDConfig.from_pretrained(__snake_case				)
						else:
												__magic_name__    :     Dict     		=      convert_config(model[0],					__snake_case				)
						__magic_name__    :     Union[str, Any]     		=      model[0].eval()

						__magic_name__    :     Optional[int]     		=      True if config.feat_extract_norm == """layer""" else False
						__magic_name__    :     Optional[Any]     		=      WavaVecaFeatureExtractor(
						    feature_size=1,					sampling_rate=16000,					padding_value=0,					do_normalize=__snake_case,					return_attention_mask=__snake_case,					)

						if is_finetuned:
												if dict_path:
																		__magic_name__    :     Dict     		=      Dictionary.load(__snake_case				)

																		# important change bos & pad token id since CTC symbol is <pad> and
																		# not <s> as in fairseq
																		__magic_name__    :     Dict     		=      target_dict.pad_index
																		__magic_name__    :     Optional[int]     		=      target_dict.bos_index
																		__magic_name__    :     List[Any]     		=      target_dict.pad_index
																		__magic_name__    :     Optional[Any]     		=      target_dict.bos_index
																		__magic_name__    :     Any     		=      target_dict.eos_index
																		__magic_name__    :     Any     		=      len(target_dict.symbols				)
																		__magic_name__    :     Any     		=      os.path.join(__snake_case,					"""vocab.json"""				)
																		if not os.path.isdir(__snake_case				):
																								logger.error("""--pytorch_dump_folder_path ({}) should be a directory""".format(__snake_case				)				)
																								return
																		os.makedirs(__snake_case,					exist_ok=__snake_case				)
																		with open(__snake_case,					"""w""",					encoding="""utf-8"""				) as vocab_handle:
																								json.dump(target_dict.indices,					__snake_case				)
																		__magic_name__    :     Any     		=      WavaVecaCTCTokenizer(
																		    __snake_case,					unk_token=target_dict.unk_word,					pad_token=target_dict.pad_word,					bos_token=target_dict.bos_word,					eos_token=target_dict.eos_word,					word_delimiter_token="""|""",					do_lower_case=__snake_case,					)
																		__magic_name__    :     int     		=      WavaVecaProcessor(feature_extractor=__snake_case,					tokenizer=__snake_case				)
																		processor.save_pretrained(__snake_case				)

												__magic_name__    :     Union[str, Any]     		=      SEWDForCTC(__snake_case				)
						else:
												__magic_name__    :     Tuple     		=      SEWDModel(__snake_case				)
												feature_extractor.save_pretrained(__snake_case				)

						recursively_load_weights(__snake_case,					__snake_case,					__snake_case				)

						hf_model.save_pretrained(__snake_case				)


if __name__ == "__main__":
		__magic_name__:  Dict  	=			argparse.ArgumentParser()
		parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
		parser.add_argument("--checkpoint_path", default=None, type=str, help="Path to fairseq checkpoint")
		parser.add_argument("--dict_path", default=None, type=str, help="Path to dict of fine-tuned model")
		parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert")
		parser.add_argument(
		    "--is_finetuned", action="store_true", help="Whether the model to convert is a fine-tuned model or not"
		)
		__magic_name__:  Union[str, Any]  	=			parser.parse_args()
		convert_sew_checkpoint(
		    args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, args.is_finetuned
		)








import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    FunnelBaseModel,
		    FunnelForMaskedLM,
		    FunnelForMultipleChoice,
		    FunnelForPreTraining,
		    FunnelForQuestionAnswering,
		    FunnelForSequenceClassification,
		    FunnelForTokenClassification,
		    FunnelModel,
		)





class  snake_case__      :
	def __init__(       self    ,     lowerCAmelCase__    ,     lowerCAmelCase__=13    ,     lowerCAmelCase__=7    ,     lowerCAmelCase__=True    ,     lowerCAmelCase__=True    ,     lowerCAmelCase__=True    ,     lowerCAmelCase__=True    ,     lowerCAmelCase__=99    ,     lowerCAmelCase__=[1, 1, 2]    ,     lowerCAmelCase__=1    ,     lowerCAmelCase__=32    ,     lowerCAmelCase__=4    ,     lowerCAmelCase__=8    ,     lowerCAmelCase__=37    ,     lowerCAmelCase__="gelu_new"    ,     lowerCAmelCase__=0.1    ,     lowerCAmelCase__=0.1    ,     lowerCAmelCase__=0.0    ,     lowerCAmelCase__=5_12    ,     lowerCAmelCase__=3    ,     lowerCAmelCase__=0.0_2    ,     lowerCAmelCase__=3    ,     lowerCAmelCase__=4    ,     lowerCAmelCase__=None    ,     lowerCAmelCase__=False    ,     )							->   Union[str, Any]:
							__magic_name__    :     Dict     		=      parent
							__magic_name__    :     List[str]     		=      batch_size
							__magic_name__    :     Dict     		=      seq_length
							__magic_name__    :     Optional[Any]     		=      is_training
							__magic_name__    :     Any     		=      use_input_mask
							__magic_name__    :     Dict     		=      use_token_type_ids
							__magic_name__    :     Tuple     		=      use_labels
							__magic_name__    :     Dict     		=      vocab_size
							__magic_name__    :     Dict     		=      block_sizes
							__magic_name__    :     Union[str, Any]     		=      num_decoder_layers
							__magic_name__    :     Dict     		=      d_model
							__magic_name__    :     List[Any]     		=      n_head
							__magic_name__    :     Any     		=      d_head
							__magic_name__    :     int     		=      d_inner
							__magic_name__    :     Optional[Any]     		=      hidden_act
							__magic_name__    :     Optional[int]     		=      hidden_dropout
							__magic_name__    :     Dict     		=      attention_dropout
							__magic_name__    :     Optional[Any]     		=      activation_dropout
							__magic_name__    :     List[Any]     		=      max_position_embeddings
							__magic_name__    :     int     		=      type_vocab_size
							__magic_name__    :     Union[str, Any]     		=      2
							__magic_name__    :     Dict     		=      num_labels
							__magic_name__    :     Dict     		=      num_choices
							__magic_name__    :     Tuple     		=      scope
							__magic_name__    :     Union[str, Any]     		=      initializer_std

							# Used in the tests to check the size of the first attention layer
							__magic_name__    :     List[Any]     		=      n_head
							# Used in the tests to check the size of the first hidden state
							__magic_name__    :     int     		=      self.d_model
							# Used in the tests to check the number of output hidden states/attentions
							__magic_name__    :     Tuple     		=      sum(self.block_sizes				) + (0 if base else self.num_decoder_layers)
							# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
							# the last hidden state of the first block (which is the first hidden state of the decoder).
							if not base:
													__magic_name__    :     Optional[int]     		=      self.num_hidden_layers + 2
	def     __magic_name__       (       self				)							->   Union[str, Any]:
							__magic_name__    :     Tuple     		=      ids_tensor([self.batch_size, self.seq_length]    ,     self.vocab_size				)

							__magic_name__    :     Tuple     		=      None
							if self.use_input_mask:
													__magic_name__    :     Any     		=      ids_tensor([self.batch_size, self.seq_length]    ,     vocab_size=2				)

							__magic_name__    :     int     		=      None
							if self.use_token_type_ids:
													__magic_name__    :     Optional[Any]     		=      ids_tensor([self.batch_size, self.seq_length]    ,     self.type_vocab_size				)

							__magic_name__    :     Tuple     		=      None
							__magic_name__    :     Optional[Any]     		=      None
							__magic_name__    :     Optional[int]     		=      None
							if self.use_labels:
													__magic_name__    :     Optional[int]     		=      ids_tensor([self.batch_size]    ,     self.type_sequence_label_size				)
													__magic_name__    :     Optional[int]     		=      ids_tensor([self.batch_size, self.seq_length]    ,     self.num_labels				)
													__magic_name__    :     Dict     		=      ids_tensor([self.batch_size]    ,     self.num_choices				)
													__magic_name__    :     Any     		=      ids_tensor([self.batch_size, self.seq_length]    ,     1				)

							__magic_name__    :     List[Any]     		=      self.get_config()

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_mask,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							    fake_token_labels,
							)
	def     __magic_name__       (       self				)							->   Optional[int]:
							return FunnelConfig(
							    vocab_size=self.vocab_size    ,     block_sizes=self.block_sizes    ,     num_decoder_layers=self.num_decoder_layers    ,     d_model=self.d_model    ,     n_head=self.n_head    ,     d_head=self.d_head    ,     d_inner=self.d_inner    ,     hidden_act=self.hidden_act    ,     hidden_dropout=self.hidden_dropout    ,     attention_dropout=self.attention_dropout    ,     activation_dropout=self.activation_dropout    ,     max_position_embeddings=self.max_position_embeddings    ,     type_vocab_size=self.type_vocab_size    ,     initializer_std=self.initializer_std    ,     )
	def     __magic_name__       (       self    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     )							->   str:
							__magic_name__    :     Optional[int]     		=      FunnelModel(config=lowerCAmelCase__				)
							model.to(lowerCAmelCase__				)
							model.eval()
							__magic_name__    :     Optional[Any]     		=      model(lowerCAmelCase__    ,     attention_mask=lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__				)
							__magic_name__    :     str     		=      model(lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__				)
							__magic_name__    :     Dict     		=      model(lowerCAmelCase__				)
							self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, self.seq_length, self.d_model)				)

							__magic_name__    :     Union[str, Any]     		=      False
							__magic_name__    :     Tuple     		=      model(lowerCAmelCase__				)
							self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, self.seq_length, self.d_model)				)

							__magic_name__    :     Optional[int]     		=      False
							__magic_name__    :     Dict     		=      model(lowerCAmelCase__				)
							self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, self.seq_length, self.d_model)				)
	def     __magic_name__       (       self    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     )							->   List[Any]:
							__magic_name__    :     List[str]     		=      FunnelBaseModel(config=lowerCAmelCase__				)
							model.to(lowerCAmelCase__				)
							model.eval()
							__magic_name__    :     Dict     		=      model(lowerCAmelCase__    ,     attention_mask=lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__				)
							__magic_name__    :     Optional[Any]     		=      model(lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__				)
							__magic_name__    :     List[str]     		=      model(lowerCAmelCase__				)
							self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, 2, self.d_model)				)

							__magic_name__    :     str     		=      False
							__magic_name__    :     Optional[int]     		=      model(lowerCAmelCase__				)
							self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, 3, self.d_model)				)

							__magic_name__    :     List[Any]     		=      False
							__magic_name__    :     Optional[int]     		=      model(lowerCAmelCase__				)
							self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, 2, self.d_model)				)
	def     __magic_name__       (       self    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     )							->   int:
							__magic_name__    :     Any     		=      self.num_labels
							__magic_name__    :     Tuple     		=      FunnelForPreTraining(config=lowerCAmelCase__				)
							model.to(lowerCAmelCase__				)
							model.eval()
							__magic_name__    :     Optional[int]     		=      model(lowerCAmelCase__    ,     attention_mask=lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__    ,     labels=lowerCAmelCase__				)
							self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.seq_length)				)
	def     __magic_name__       (       self    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     )							->   List[str]:
							__magic_name__    :     int     		=      FunnelForMaskedLM(config=lowerCAmelCase__				)
							model.to(lowerCAmelCase__				)
							model.eval()
							__magic_name__    :     Dict     		=      model(lowerCAmelCase__    ,     attention_mask=lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__    ,     labels=lowerCAmelCase__				)
							self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.seq_length, self.vocab_size)				)
	def     __magic_name__       (       self    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     )							->   Any:
							__magic_name__    :     Tuple     		=      self.num_labels
							__magic_name__    :     str     		=      FunnelForSequenceClassification(lowerCAmelCase__				)
							model.to(lowerCAmelCase__				)
							model.eval()
							__magic_name__    :     int     		=      model(lowerCAmelCase__    ,     attention_mask=lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__    ,     labels=lowerCAmelCase__				)
							self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.num_labels)				)
	def     __magic_name__       (       self    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     )							->   str:
							__magic_name__    :     str     		=      self.num_choices
							__magic_name__    :     int     		=      FunnelForMultipleChoice(config=lowerCAmelCase__				)
							model.to(lowerCAmelCase__				)
							model.eval()
							__magic_name__    :     Union[str, Any]     		=      input_ids.unsqueeze(1				).expand(-1    ,     self.num_choices    ,     -1				).contiguous()
							__magic_name__    :     List[Any]     		=      token_type_ids.unsqueeze(1				).expand(-1    ,     self.num_choices    ,     -1				).contiguous()
							__magic_name__    :     Optional[Any]     		=      input_mask.unsqueeze(1				).expand(-1    ,     self.num_choices    ,     -1				).contiguous()
							__magic_name__    :     Union[str, Any]     		=      model(
							    lowerCAmelCase__    ,     attention_mask=lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__    ,     labels=lowerCAmelCase__    ,     )
							self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.num_choices)				)
	def     __magic_name__       (       self    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     )							->   Tuple:
							__magic_name__    :     Tuple     		=      self.num_labels
							__magic_name__    :     int     		=      FunnelForTokenClassification(config=lowerCAmelCase__				)
							model.to(lowerCAmelCase__				)
							model.eval()
							__magic_name__    :     Union[str, Any]     		=      model(lowerCAmelCase__    ,     attention_mask=lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__    ,     labels=lowerCAmelCase__				)
							self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.seq_length, self.num_labels)				)
	def     __magic_name__       (       self    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     )							->   Optional[int]:
							__magic_name__    :     Union[str, Any]     		=      FunnelForQuestionAnswering(config=lowerCAmelCase__				)
							model.to(lowerCAmelCase__				)
							model.eval()
							__magic_name__    :     Optional[int]     		=      model(
							    lowerCAmelCase__    ,     attention_mask=lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__    ,     start_positions=lowerCAmelCase__    ,     end_positions=lowerCAmelCase__    ,     )
							self.parent.assertEqual(result.start_logits.shape    ,     (self.batch_size, self.seq_length)				)
							self.parent.assertEqual(result.end_logits.shape    ,     (self.batch_size, self.seq_length)				)







	def     __magic_name__       (       self				)							->   List[Any]:
							__magic_name__    :     Tuple     		=      self.prepare_config_and_inputs()
							(
							    (
							    __magic_name__
							)		,(
							    __magic_name__
							)		,(
							    __magic_name__
							)		,(
							    __magic_name__
							)		,(
							    __magic_name__
							)		,(
							    __magic_name__
							)		,(
							    __magic_name__
							)		,(
							    __magic_name__
							)		,
							)    :     str     		=      config_and_inputs
							__magic_name__    :     Optional[Any]     		=      {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
							return config, inputs_dict





@require_torch
class  snake_case__      (							_lowerCAmelCase		,						_lowerCAmelCase		,						unittest.TestCase   ):
	lowercase__		:  Optional[int]							   =					False
	lowercase__		:  Dict							   =					False
	lowercase__		:  int							   =					(
	    (
	        FunnelModel,
	        FunnelForMaskedLM,
	        FunnelForPreTraining,
	        FunnelForQuestionAnswering,
	        FunnelForTokenClassification,
	    )
	    if is_torch_available()
	    else ()
	)
	lowercase__		:  Optional[int]							   =					(
	    {
	        '''feature-extraction''': (FunnelBaseModel, FunnelModel),
	        '''fill-mask''': FunnelForMaskedLM,
	        '''question-answering''': FunnelForQuestionAnswering,
	        '''text-classification''': FunnelForSequenceClassification,
	        '''token-classification''': FunnelForTokenClassification,
	        '''zero-shot''': FunnelForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	def     __magic_name__       (       self    ,     lowerCAmelCase__    ,     lowerCAmelCase__    ,     lowerCAmelCase__=False				)							->   Any:
							__magic_name__    :     Union[str, Any]     		=      super()._prepare_for_class(lowerCAmelCase__    ,     lowerCAmelCase__    ,     return_labels=lowerCAmelCase__				)

							if return_labels:
													if model_class in get_values(lowerCAmelCase__				):
																			__magic_name__    :     List[Any]     		=      torch.zeros(
																			    (self.model_tester.batch_size, self.model_tester.seq_length)    ,     dtype=torch.long    ,     device=lowerCAmelCase__				)
							return inputs_dict
	def     __magic_name__       (       self				)							->   Optional[Any]:
							__magic_name__    :     Any     		=      FunnelModelTester(self				)
							__magic_name__    :     Union[str, Any]     		=      ConfigTester(self    ,     config_class=lowerCAmelCase__				)
	def     __magic_name__       (       self				)							->   List[Any]:
							self.config_tester.run_common_tests()
	def     __magic_name__       (       self				)							->   Any:
							__magic_name__    :     Optional[Any]     		=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*lowerCAmelCase__				)
	def     __magic_name__       (       self				)							->   Any:
							__magic_name__    :     Optional[Any]     		=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase__				)
	def     __magic_name__       (       self				)							->   Tuple:
							__magic_name__    :     Optional[int]     		=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__				)
	def     __magic_name__       (       self				)							->   Tuple:
							__magic_name__    :     Tuple     		=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__				)
	def     __magic_name__       (       self				)							->   Union[str, Any]:
							__magic_name__    :     int     		=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__				)







	def     __magic_name__       (       self    ,     lowerCAmelCase__				)							->   int:
							if hasattr(lowerCAmelCase__    ,     """weight"""				) and module.weight is not None:
													module.weight.data.fill_(3				)
							if hasattr(lowerCAmelCase__    ,     """bias"""				) and module.bias is not None:
													module.bias.data.fill_(3				)

							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
													if hasattr(lowerCAmelCase__    ,     lowerCAmelCase__				) and getattr(lowerCAmelCase__    ,     lowerCAmelCase__				) is not None:
																			__magic_name__    :     Optional[int]     		=      getattr(lowerCAmelCase__    ,     lowerCAmelCase__				)
																			weight.data.fill_(3				)





@require_torch
class  snake_case__      (							_lowerCAmelCase		,						unittest.TestCase   ):
	lowercase__		:  Tuple							   =					False
	lowercase__		:  int							   =					False
	lowercase__		:  Optional[int]							   =					(
	    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
	)
	def     __magic_name__       (       self				)							->   Optional[Any]:
							__magic_name__    :     str     		=      FunnelModelTester(self    ,     base=lowerCAmelCase__				)
							__magic_name__    :     Tuple     		=      ConfigTester(self    ,     config_class=lowerCAmelCase__				)
	def     __magic_name__       (       self				)							->   str:
							self.config_tester.run_common_tests()
	def     __magic_name__       (       self				)							->   Optional[int]:
							__magic_name__    :     List[str]     		=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_base_model(*lowerCAmelCase__				)
	def     __magic_name__       (       self				)							->   str:
							__magic_name__    :     Any     		=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__				)
	def     __magic_name__       (       self				)							->   Tuple:
							__magic_name__    :     Any     		=      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase__				)
	def     __magic_name__       (       self				)							->   List[str]:
							__magic_name__		,__magic_name__    :     Tuple     		=      self.model_tester.prepare_config_and_inputs_for_common()
							__magic_name__    :     int     		=      True

							for model_class in self.all_model_classes:
													if model_class.__name__ == "FunnelBaseModel":
																			continue
													__magic_name__    :     Tuple     		=      model_class(lowerCAmelCase__				)
													model.to(lowerCAmelCase__				)
													model.train()
													__magic_name__    :     Tuple     		=      self._prepare_for_class(lowerCAmelCase__    ,     lowerCAmelCase__    ,     return_labels=lowerCAmelCase__				)
													__magic_name__    :     Dict     		=      model(**lowerCAmelCase__				).loss
													loss.backward()







	def     __magic_name__       (       self    ,     lowerCAmelCase__				)							->   Optional[int]:
							if hasattr(lowerCAmelCase__    ,     """weight"""				) and module.weight is not None:
													module.weight.data.fill_(3				)
							if hasattr(lowerCAmelCase__    ,     """bias"""				) and module.bias is not None:
													module.bias.data.fill_(3				)

							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
													if hasattr(lowerCAmelCase__    ,     lowerCAmelCase__				) and getattr(lowerCAmelCase__    ,     lowerCAmelCase__				) is not None:
																			__magic_name__    :     Tuple     		=      getattr(lowerCAmelCase__    ,     lowerCAmelCase__				)
																			weight.data.fill_(3				)





@require_torch
@require_sentencepiece
@require_tokenizers
class  snake_case__      (							unittest.TestCase   ):
	def     __magic_name__       (       self				)							->   List[Any]:
							__magic_name__    :     Union[str, Any]     		=      13
							__magic_name__    :     str     		=      7
							__magic_name__    :     Optional[int]     		=      torch.arange(0    ,     batch_size * sequence_length				).long().reshape(lowerCAmelCase__    ,     lowerCAmelCase__				)
							__magic_name__    :     str     		=      [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
							__magic_name__    :     List[Any]     		=      torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]				)

							__magic_name__    :     Optional[int]     		=      FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""				)
							__magic_name__    :     List[str]     		=      model(lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__				)[0].abs()

							__magic_name__    :     Tuple     		=      torch.tensor(2_3_4_4.8_3_5_2				)
							__magic_name__    :     Optional[Any]     		=      torch.tensor(0.8_0_5_2				)
							self.assertTrue(torch.allclose(output.sum()    ,     lowerCAmelCase__    ,     atol=1e-4				)				)
							self.assertTrue(torch.allclose(output.mean()    ,     lowerCAmelCase__    ,     atol=1e-4				)				)

							__magic_name__    :     int     		=      torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]				)
							__magic_name__    :     Tuple     		=      model(lowerCAmelCase__    ,     attention_mask=lowerCAmelCase__    ,     token_type_ids=lowerCAmelCase__				)[0].abs()

							__magic_name__    :     List[Any]     		=      torch.tensor(2_3_4_3.8_4_2_5				)
							__magic_name__    :     Dict     		=      torch.tensor(0.8_0_4_9				)
							self.assertTrue(torch.allclose(output.sum()    ,     lowerCAmelCase__    ,     atol=1e-4				)				)
							self.assertTrue(torch.allclose(output.mean()    ,     lowerCAmelCase__    ,     atol=1e-4				)				)







	@slow
	def     __magic_name__       (       self				)							->   List[Any]:
							__magic_name__    :     Tuple     		=      FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""				)
							__magic_name__    :     Union[str, Any]     		=      FunnelModel.from_pretrained("""huggingface/funnel-small"""				)
							__magic_name__    :     str     		=      tokenizer("""Hello! I am the Funnel Transformer model."""    ,     return_tensors="""pt"""				)
							__magic_name__    :     Optional[Any]     		=      model(**lowerCAmelCase__				)[0]

							__magic_name__    :     List[Any]     		=      torch.tensor(2_3_5.7_2_4_6				)
							__magic_name__    :     List[Any]     		=      torch.tensor(0.0_2_5_6				)
							self.assertTrue(torch.allclose(output.sum()    ,     lowerCAmelCase__    ,     atol=1e-4				)				)
							self.assertTrue(torch.allclose(output.mean()    ,     lowerCAmelCase__    ,     atol=1e-4				)				)
