


"""simple docstring"""
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
				__SCREAMING_SNAKE_CASE				:     Optional[int]		=     None


__SCREAMING_SNAKE_CASE				:     Optional[int]		=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE				:     Optional[int]		=     {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE				:     List[str]		=     {
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'
        ),
    },
}

__SCREAMING_SNAKE_CASE				:     List[str]		=     {
    'xlm-roberta-base': 512,
    'xlm-roberta-large': 512,
    'xlm-roberta-large-finetuned-conll02-dutch': 512,
    'xlm-roberta-large-finetuned-conll02-spanish': 512,
    'xlm-roberta-large-finetuned-conll03-english': 512,
    'xlm-roberta-large-finetuned-conll03-german': 512,
}





class   __A  (__A):





							'''simple docstring'''

							__lowercase:		Optional[Any]				   =     VOCAB_FILES_NAMES
							__lowercase:		Optional[Any]				   =     PRETRAINED_VOCAB_FILES_MAP
							__lowercase:		Dict				   =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase:		str				   =     ["input_ids", "attention_mask"]
							__lowercase:		str				   =     XLMRobertaTokenizer



							def __init__(					self				:   Any		,							UpperCAmelCase_				:   List[str]=None		,							UpperCAmelCase_				:   Optional[int]=None		,							UpperCAmelCase_				:   Union[str, Any]="<s>"		,							UpperCAmelCase_				:   List[str]="</s>"		,							UpperCAmelCase_				:   Tuple="</s>"		,							UpperCAmelCase_				:   Optional[Any]="<s>"		,							UpperCAmelCase_				:   Optional[Any]="<unk>"		,							UpperCAmelCase_				:   Any="<pad>"		,							UpperCAmelCase_				:   Union[str, Any]="<mask>"		,							**UpperCAmelCase_				:   Optional[Any]		,							)       ->List[Any]:



														"""simple docstring"""



														snake_case_     							=		AddedToken(lowercase_		,							lstrip=lowercase_		,							rstrip=lowercase_							) if isinstance(lowercase_		,							lowercase_							) else mask_token

														super().__init__(
														    lowercase_		,							tokenizer_file=lowercase_		,							bos_token=lowercase_		,							eos_token=lowercase_		,							sep_token=lowercase_		,							cls_token=lowercase_		,							unk_token=lowercase_		,							pad_token=lowercase_		,							mask_token=lowercase_		,							**lowercase_		,							)

														snake_case_     							=		vocab_file
														snake_case_     							=		False if not self.vocab_file else True



							def 		lowerCAmelCase							(					self				:   Any		,							UpperCAmelCase_				:   List[int]		,							UpperCAmelCase_				:   Optional[List[int]] = None							)       ->List[int]:



														"""simple docstring"""



														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														snake_case_     							=		[self.cls_token_id]
														snake_case_     							=		[self.sep_token_id]
														return cls + token_ids_a + sep + sep + token_ids_a + sep



							def 		lowerCAmelCase							(					self				:   Union[str, Any]		,							UpperCAmelCase_				:   List[int]		,							UpperCAmelCase_				:   Optional[List[int]] = None							)       ->List[int]:



														"""simple docstring"""



														snake_case_     							=		[self.sep_token_id]
														snake_case_     							=		[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep							) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]



							def 		lowerCAmelCase							(					self				:   List[str]		,							UpperCAmelCase_				:   str		,							UpperCAmelCase_				:   Optional[str] = None							)       ->Tuple[str]:



														"""simple docstring"""



														if not self.can_save_slow_tokenizer:
																					raise ValueError(
																					    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
																					    """tokenizer."""							)

														if not os.path.isdir(lowercase_							):
																					logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""							)
																					return
														snake_case_     							=		os.path.join(
														    lowercase_		,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

														if os.path.abspath(self.vocab_file							) != os.path.abspath(lowercase_							):
																					copyfile(self.vocab_file		,							lowercase_							)

														return (out_vocab_file,)




from __future__ import annotations

import unittest

from transformers import MPNetConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers.models.mpnet.modeling_tf_mpnet import (
          TFMPNetForMaskedLM,
          TFMPNetForMultipleChoice,
          TFMPNetForQuestionAnswering,
          TFMPNetForSequenceClassification,
          TFMPNetForTokenClassification,
          TFMPNetModel,
      )



class     snake_case_			:






   def __init__( self							:	Optional[Any]      ,		lowercase_							:	Tuple      ,		lowercase_							:	List[str]=13      ,		lowercase_							:	str=7      ,		lowercase_							:	str=True      ,		lowercase_							:	Dict=True      ,		lowercase_							:	List[str]=False      ,		lowercase_							:	Optional[Any]=True      ,		lowercase_							:	List[str]=99      ,		lowercase_							:	Optional[int]=64      ,		lowercase_							:	Optional[int]=2      ,		lowercase_							:	List[Any]=4      ,		lowercase_							:	Union[str, Any]=64      ,		lowercase_							:	Optional[Any]="gelu"      ,		lowercase_							:	str=0.1      ,		lowercase_							:	Union[str, Any]=0.1      ,		lowercase_							:	Optional[Any]=5_12      ,		lowercase_							:	Optional[Any]=16      ,		lowercase_							:	Optional[Any]=2      ,		lowercase_							:	Any=0.02      ,		lowercase_							:	List[Any]=3      ,		lowercase_							:	List[str]=4      ,		lowercase_							:	Union[str, Any]=None      ,		)   ->		Optional[int]:
       lowercase__					:  str          =							parent
       lowercase__					:  List[str]          =							batch_size
       lowercase__					:  Optional[int]          =							seq_length
       lowercase__					:  List[str]          =							is_training
       lowercase__					:  Union[str, Any]          =							use_input_mask
       lowercase__					:  Tuple          =							use_token_type_ids
       lowercase__					:  str          =							use_labels
       lowercase__					:  int          =							vocab_size
       lowercase__					:  Optional[Any]          =							hidden_size
       lowercase__					:  Dict          =							num_hidden_layers
       lowercase__					:  Any          =							num_attention_heads
       lowercase__					:  Dict          =							intermediate_size
       lowercase__					:  List[Any]          =							hidden_act
       lowercase__					:  int          =							hidden_dropout_prob
       lowercase__					:  Dict          =							attention_probs_dropout_prob
       lowercase__					:  Union[str, Any]          =							max_position_embeddings
       lowercase__					:  List[Any]          =							type_vocab_size
       lowercase__					:  str          =							type_sequence_label_size
       lowercase__					:  Union[str, Any]          =							initializer_range
       lowercase__					:  str          =							num_labels
       lowercase__					:  List[Any]          =							num_choices
       lowercase__					:  List[str]          =							scope






   def  __UpperCamelCase	( self							:	Dict    )   ->		Tuple:
       lowercase__					:  int          =							ids_tensor([self.batch_size, self.seq_length]      ,		self.vocab_size    )

       lowercase__					:  List[str]          =							None
       if self.use_input_mask:
           lowercase__					:  str          =							random_attention_mask([self.batch_size, self.seq_length]    )

       lowercase__					:  Any          =							None
       lowercase__					:  Optional[Any]          =							None
       lowercase__					:  Optional[int]          =							None
       if self.use_labels:
           lowercase__					:  str          =							ids_tensor([self.batch_size]      ,		self.type_sequence_label_size    )
           lowercase__					:  Optional[Any]          =							ids_tensor([self.batch_size, self.seq_length]      ,		self.num_labels    )
           lowercase__					:  Any          =							ids_tensor([self.batch_size]      ,		self.num_choices    )

       lowercase__					:  Any          =							MPNetConfig(
           vocab_size=self.vocab_size      ,		hidden_size=self.hidden_size      ,		num_hidden_layers=self.num_hidden_layers      ,		num_attention_heads=self.num_attention_heads      ,		intermediate_size=self.intermediate_size      ,		hidden_act=self.hidden_act      ,		hidden_dropout_prob=self.hidden_dropout_prob      ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,		max_position_embeddings=self.max_position_embeddings      ,		initializer_range=self.initializer_range      ,		)
       return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






   def  __UpperCamelCase	( self							:	int      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	Union[str, Any]      ,		lowercase_							:	Any      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	Any      ,		lowercase_							:	List[Any]    )   ->		List[str]:
       lowercase__					:  int          =							TFMPNetModel(config=lowercase_    )
       lowercase__					:  Dict          =							{"input_ids": input_ids, "attention_mask": input_mask}
       lowercase__					:  Dict          =							model(lowercase_    )
       lowercase__					:  Tuple          =							[input_ids, input_mask]
       lowercase__					:  Any          =							model(lowercase_    )
       self.parent.assertEqual(result.last_hidden_state.shape      ,		(self.batch_size, self.seq_length, self.hidden_size)    )






   def  __UpperCamelCase	( self							:	List[Any]      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Tuple      ,		lowercase_							:	str      ,		lowercase_							:	Union[str, Any]      ,		lowercase_							:	Union[str, Any]      ,		lowercase_							:	List[Any]    )   ->		Union[str, Any]:
       lowercase__					:  List[str]          =							TFMPNetForMaskedLM(config=lowercase_    )
       lowercase__					:  Dict          =							{"input_ids": input_ids, "attention_mask": input_mask}
       lowercase__					:  Optional[int]          =							model(lowercase_    )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length, self.vocab_size)    )






   def  __UpperCamelCase	( self							:	Optional[Any]      ,		lowercase_							:	List[Any]      ,		lowercase_							:	Union[str, Any]      ,		lowercase_							:	Dict      ,		lowercase_							:	Tuple      ,		lowercase_							:	Tuple      ,		lowercase_							:	Optional[Any]    )   ->		List[Any]:
       lowercase__					:  List[Any]          =							TFMPNetForQuestionAnswering(config=lowercase_    )
       lowercase__					:  Optional[int]          =							{
           "input_ids": input_ids,
           "attention_mask": input_mask,
       }
       lowercase__					:  int          =							model(lowercase_    )
       self.parent.assertEqual(result.start_logits.shape      ,		(self.batch_size, self.seq_length)    )
       self.parent.assertEqual(result.end_logits.shape      ,		(self.batch_size, self.seq_length)    )






   def  __UpperCamelCase	( self							:	Union[str, Any]      ,		lowercase_							:	List[str]      ,		lowercase_							:	List[Any]      ,		lowercase_							:	Tuple      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Any    )   ->		int:
       lowercase__					:  Optional[Any]          =							self.num_labels
       lowercase__					:  int          =							TFMPNetForSequenceClassification(lowercase_    )
       lowercase__					:  Any          =							{"input_ids": input_ids, "attention_mask": input_mask}
       lowercase__					:  Optional[Any]          =							model(lowercase_    )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.num_labels)    )






   def  __UpperCamelCase	( self							:	List[str]      ,		lowercase_							:	int      ,		lowercase_							:	Optional[int]      ,		lowercase_							:	Union[str, Any]      ,		lowercase_							:	Optional[Any]      ,		lowercase_							:	List[str]      ,		lowercase_							:	List[str]    )   ->		Optional[int]:
       lowercase__					:  List[Any]          =							self.num_choices
       lowercase__					:  List[str]          =							TFMPNetForMultipleChoice(lowercase_    )
       lowercase__					:  Tuple          =							tf.tile(tf.expand_dims(lowercase_      ,		1    )      ,		(1, self.num_choices, 1)    )
       lowercase__					:  List[str]          =							tf.tile(tf.expand_dims(lowercase_      ,		1    )      ,		(1, self.num_choices, 1)    )
       lowercase__					:  Tuple          =							{
           "input_ids": multiple_choice_inputs_ids,
           "attention_mask": multiple_choice_input_mask,
       }
       lowercase__					:  List[str]          =							model(lowercase_    )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.num_choices)    )






   def  __UpperCamelCase	( self							:	Tuple      ,		lowercase_							:	Tuple      ,		lowercase_							:	str      ,		lowercase_							:	List[Any]      ,		lowercase_							:	str      ,		lowercase_							:	str      ,		lowercase_							:	Tuple    )   ->		int:
       lowercase__					:  Dict          =							self.num_labels
       lowercase__					:  int          =							TFMPNetForTokenClassification(lowercase_    )
       lowercase__					:  List[str]          =							{"input_ids": input_ids, "attention_mask": input_mask}
       lowercase__					:  Dict          =							model(lowercase_    )
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length, self.num_labels)    )







   def  __UpperCamelCase	( self							:	List[Any]    )   ->		List[str]:
       lowercase__					:  int          =							self.prepare_config_and_inputs()
       ((lowercase__)   ,  (lowercase__)   ,  (lowercase__)   ,  (lowercase__)   ,  (lowercase__)   ,  (lowercase__))					:  str          =							config_and_inputs
       lowercase__					:  Union[str, Any]          =							{"input_ids": input_ids, "attention_mask": input_mask}
       return config, inputs_dict



@require_tf
class     snake_case_			(						__A    ,__A    ,unittest.TestCase			):
   __A      :						List[str]   =							(
       (
           TFMPNetForMaskedLM,
           TFMPNetForMultipleChoice,
           TFMPNetForQuestionAnswering,
           TFMPNetForSequenceClassification,
           TFMPNetForTokenClassification,
           TFMPNetModel,
       )
       if is_tf_available()
       else ()
   )
   __A      :						int   =							(
       {
           "feature-extraction": TFMPNetModel,
           "fill-mask": TFMPNetForMaskedLM,
           "question-answering": TFMPNetForQuestionAnswering,
           "text-classification": TFMPNetForSequenceClassification,
           "token-classification": TFMPNetForTokenClassification,
           "zero-shot": TFMPNetForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   __A      :						str   =							False
   __A      :						Union[str, Any]   =							False






   def  __UpperCamelCase	( self							:	List[Any]    )   ->		List[str]:
       lowercase__					:  Optional[int]          =							TFMPNetModelTester(self    )
       lowercase__					:  List[Any]          =							ConfigTester(self      ,		config_class=lowercase_      ,		hidden_size=37    )






   def  __UpperCamelCase	( self							:	int    )   ->		Dict:
       self.config_tester.run_common_tests()






   def  __UpperCamelCase	( self							:	Any    )   ->		Any:
       lowercase__					:  Tuple          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mpnet_model(*lowercase_    )






   def  __UpperCamelCase	( self							:	Optional[Any]    )   ->		Dict:
       lowercase__					:  str          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mpnet_for_masked_lm(*lowercase_    )






   def  __UpperCamelCase	( self							:	Optional[int]    )   ->		Any:
       lowercase__					:  Optional[Any]          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mpnet_for_question_answering(*lowercase_    )






   def  __UpperCamelCase	( self							:	int    )   ->		List[Any]:
       lowercase__					:  Dict          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mpnet_for_sequence_classification(*lowercase_    )






   def  __UpperCamelCase	( self							:	Dict    )   ->		Any:
       lowercase__					:  List[Any]          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mpnet_for_multiple_choice(*lowercase_    )






   def  __UpperCamelCase	( self							:	List[Any]    )   ->		Dict:
       lowercase__					:  List[Any]          =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mpnet_for_token_classification(*lowercase_    )







   @slow
   def  __UpperCamelCase	( self							:	Dict    )   ->		int:
       for model_name in ["microsoft/mpnet-base"]:
           lowercase__					:  List[str]          =							TFMPNetModel.from_pretrained(lowercase_    )
           self.assertIsNotNone(lowercase_    )



@require_tf
class     snake_case_			(						unittest.TestCase			):






   @slow
   def  __UpperCamelCase	( self							:	int    )   ->		List[Any]:
       lowercase__					:  Optional[Any]          =							TFMPNetModel.from_pretrained("microsoft/mpnet-base"    )
       lowercase__					:  Tuple          =							tf.constant([[0, 1, 2, 3, 4, 5]]    )
       lowercase__					:  List[str]          =							model(lowercase_    )[0]

       lowercase__					:  List[Any]          =							[1, 6, 7_68]
       self.assertEqual(output.shape      ,		lowercase_    )

       lowercase__					:  Optional[int]          =							tf.constant(
           [
               [
                   [-0.1_06_71_72, 0.08_21_64_73, 0.0_02_45_43],
                   [-0.03_46_58_79, 0.8_35_41_18, -0.03_25_22_88],
                   [-0.06_56_94_76, -0.12_42_41_11, -0.0_49_44_36],
               ]
           ]    )
       tf.debugging.assert_near(output[:, :3, :3]      ,		lowercase_      ,		atol=1E-4    )



