

"""simple docstring"""
import unittest

from transformers import ErnieMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     ErnieMForInformationExtraction,
     ErnieMForMultipleChoice,
     ErnieMForQuestionAnswering,
     ErnieMForSequenceClassification,
     ErnieMForTokenClassification,
     ErnieMModel,
 )
 from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST

class        _UpperCamelCase	:


  """simple docstring"""





  def __init__( self					: Dict       ,       snake_case					: Dict       ,       snake_case					: Optional[Any]=13       ,       snake_case					: Optional[int]=7       ,       snake_case					: Any=True       ,       snake_case					: Tuple=True       ,       snake_case					: Any=True       ,       snake_case					: List[Any]=99       ,       snake_case					: Union[str, Any]=32       ,       snake_case					: List[str]=5       ,       snake_case					: Any=4       ,       snake_case					: Union[str, Any]=37       ,       snake_case					: Optional[int]="gelu"       ,       snake_case					: str=0.1       ,       snake_case					: Dict=0.1       ,       snake_case					: Any=512       ,       snake_case					: List[Any]=16       ,       snake_case					: Any=2       ,       snake_case					: Union[str, Any]=0.02       ,       snake_case					: Dict=3       ,       snake_case					: Tuple=4       ,       snake_case					: List[Any]=None       ,       )	->			Tuple:


         '''simple docstring'''





         __magic_name__			:		Dict        =     parent
         __magic_name__			:		List[str]        =     batch_size
         __magic_name__			:		List[str]        =     seq_length
         __magic_name__			:		str        =     is_training
         __magic_name__			:		Tuple        =     use_input_mask
         __magic_name__			:		Tuple        =     use_labels
         __magic_name__			:		int        =     vocab_size
         __magic_name__			:		Optional[Any]        =     hidden_size
         __magic_name__			:		int        =     num_hidden_layers
         __magic_name__			:		List[Any]        =     num_attention_heads
         __magic_name__			:		Dict        =     intermediate_size
         __magic_name__			:		str        =     hidden_act
         __magic_name__			:		int        =     hidden_dropout_prob
         __magic_name__			:		Optional[Any]        =     attention_probs_dropout_prob
         __magic_name__			:		List[str]        =     max_position_embeddings
         __magic_name__			:		Optional[int]        =     type_vocab_size
         __magic_name__			:		Tuple        =     type_sequence_label_size
         __magic_name__			:		Any        =     initializer_range
         __magic_name__			:		List[Any]        =     num_labels
         __magic_name__			:		List[Any]        =     num_choices
         __magic_name__			:		List[str]        =     scope



  def 							_UpperCAmelCase      ( self					: int      )	->			Any:


         '''simple docstring'''





         __magic_name__			:		Dict        =     ids_tensor([self.batch_size, self.seq_length]       ,       self.vocab_size      )

         __magic_name__			:		Optional[int]        =     None
         if self.use_input_mask:
                __magic_name__			:		List[str]        =     random_attention_mask([self.batch_size, self.seq_length]      )

         __magic_name__			:		List[Any]        =     None
         __magic_name__			:		Tuple        =     None
         __magic_name__			:		Union[str, Any]        =     None
         if self.use_labels:
                __magic_name__			:		Any        =     ids_tensor([self.batch_size]       ,       self.type_sequence_label_size      )
                __magic_name__			:		Tuple        =     ids_tensor([self.batch_size, self.seq_length]       ,       self.num_labels      )
                __magic_name__			:		Any        =     ids_tensor([self.batch_size]       ,       self.num_choices      )

         __magic_name__			:		Optional[int]        =     self.get_config()

         return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels



  def 							_UpperCAmelCase      ( self					: int      )	->			Optional[int]:


         '''simple docstring'''





         __magic_name__			:		Optional[Any]        =     ids_tensor([self.batch_size, self.seq_length]       ,       self.vocab_size      )

         __magic_name__			:		int        =     None
         if self.use_input_mask:
                __magic_name__			:		List[str]        =     random_attention_mask([self.batch_size, self.seq_length]      )
         __magic_name__			:		Union[str, Any]        =     self.get_config()

         return config, input_ids, input_mask



  def 							_UpperCAmelCase      ( self					: List[Any]      )	->			List[str]:


         '''simple docstring'''





         return ErnieMConfig(
             vocab_size=self.vocab_size       ,       hidden_size=self.hidden_size       ,       num_hidden_layers=self.num_hidden_layers       ,       num_attention_heads=self.num_attention_heads       ,       intermediate_size=self.intermediate_size       ,       hidden_act=self.hidden_act       ,       hidden_dropout_prob=self.hidden_dropout_prob       ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,       max_position_embeddings=self.max_position_embeddings       ,       type_vocab_size=self.type_vocab_size       ,       initializer_range=self.initializer_range       ,       )



  def 							_UpperCAmelCase      ( self					: List[str]       ,       snake_case					: Tuple       ,       snake_case					: Optional[int]       ,       snake_case					: Dict       ,       snake_case					: Dict       ,       snake_case					: Tuple       ,       snake_case					: str      )	->			str:


         '''simple docstring'''





         __magic_name__			:		List[str]        =     ErnieMModel(config=snake_case      )
         model.to(snake_case      )
         model.eval()
         __magic_name__			:		Union[str, Any]        =     model(snake_case       ,       return_dict=snake_case      )
         self.parent.assertEqual(result.last_hidden_state.shape       ,       (self.batch_size, self.seq_length, self.hidden_size)      )



  def 							_UpperCAmelCase      ( self					: int       ,       snake_case					: Optional[int]       ,       snake_case					: Union[str, Any]       ,       snake_case					: Tuple       ,       snake_case					: Any       ,       snake_case					: str       ,       snake_case					: Union[str, Any]      )	->			Union[str, Any]:


         '''simple docstring'''





         __magic_name__			:		Tuple        =     ErnieMForQuestionAnswering(config=snake_case      )
         model.to(snake_case      )
         model.eval()
         __magic_name__			:		Optional[Any]        =     model(
             snake_case       ,       attention_mask=snake_case       ,       start_positions=snake_case       ,       end_positions=snake_case       ,       )
         self.parent.assertEqual(result.start_logits.shape       ,       (self.batch_size, self.seq_length)      )
         self.parent.assertEqual(result.end_logits.shape       ,       (self.batch_size, self.seq_length)      )



  def 							_UpperCAmelCase      ( self					: Any       ,       snake_case					: List[Any]       ,       snake_case					: Dict       ,       snake_case					: Dict       ,       snake_case					: Optional[int]       ,       snake_case					: Tuple       ,       snake_case					: Optional[int]      )	->			str:


         '''simple docstring'''





         __magic_name__			:		Optional[int]        =     ErnieMForInformationExtraction(config=snake_case      )
         model.to(snake_case      )
         model.eval()
         __magic_name__			:		Tuple        =     torch.ones_like(snake_case       ,       dtype=torch.floataa      )
         __magic_name__			:		Optional[Any]        =     model(
             snake_case       ,       attention_mask=snake_case       ,       start_positions=snake_case       ,       end_positions=snake_case       ,       )
         self.parent.assertEqual(result.start_logits.shape       ,       (self.batch_size, self.seq_length)      )
         self.parent.assertEqual(result.end_logits.shape       ,       (self.batch_size, self.seq_length)      )



  def 							_UpperCAmelCase      ( self					: int       ,       snake_case					: Dict       ,       snake_case					: Any       ,       snake_case					: Union[str, Any]       ,       snake_case					: List[Any]       ,       snake_case					: int       ,       snake_case					: Optional[int]      )	->			str:


         '''simple docstring'''





         __magic_name__			:		Optional[int]        =     self.num_labels
         __magic_name__			:		Any        =     ErnieMForSequenceClassification(snake_case      )
         model.to(snake_case      )
         model.eval()
         __magic_name__			:		Union[str, Any]        =     model(snake_case       ,       attention_mask=snake_case       ,       labels=snake_case      )
         self.parent.assertEqual(result.logits.shape       ,       (self.batch_size, self.num_labels)      )



  def 							_UpperCAmelCase      ( self					: str       ,       snake_case					: Union[str, Any]       ,       snake_case					: str       ,       snake_case					: List[str]       ,       snake_case					: int       ,       snake_case					: str       ,       snake_case					: Union[str, Any]      )	->			int:


         '''simple docstring'''





         __magic_name__			:		List[str]        =     self.num_labels
         __magic_name__			:		Optional[Any]        =     ErnieMForTokenClassification(config=snake_case      )
         model.to(snake_case      )
         model.eval()
         input_ids.to(snake_case      )
         input_mask.to(snake_case      )
         token_labels.to(snake_case      )

         __magic_name__			:		List[Any]        =     model(snake_case       ,       attention_mask=snake_case       ,       labels=snake_case      )

         self.parent.assertEqual(result.logits.shape       ,       (self.batch_size, self.seq_length, self.num_labels)      )



  def 							_UpperCAmelCase      ( self					: Any       ,       snake_case					: Dict       ,       snake_case					: Optional[int]       ,       snake_case					: str       ,       snake_case					: List[str]       ,       snake_case					: int       ,       snake_case					: List[Any]      )	->			Union[str, Any]:


         '''simple docstring'''





         __magic_name__			:		int        =     self.num_choices
         __magic_name__			:		Optional[Any]        =     ErnieMForMultipleChoice(config=snake_case      )
         model.to(snake_case      )
         model.eval()
         __magic_name__			:		Union[str, Any]        =     input_ids.unsqueeze(1      ).expand(-1       ,       self.num_choices       ,       -1      ).contiguous()
         __magic_name__			:		List[str]        =     input_mask.unsqueeze(1      ).expand(-1       ,       self.num_choices       ,       -1      ).contiguous()
         __magic_name__			:		Any        =     model(
             snake_case       ,       attention_mask=snake_case       ,       labels=snake_case       ,       )
         self.parent.assertEqual(result.logits.shape       ,       (self.batch_size, self.num_choices)      )




  def 							_UpperCAmelCase      ( self					: Optional[int]      )	->			Tuple:


         '''simple docstring'''





         __magic_name__			:		Optional[Any]        =     self.prepare_config_and_inputs()
         (
             (
             __magic_name__      
         )       ,							(
             __magic_name__      
         )       ,							(
             __magic_name__      
         )       ,							(
             __magic_name__      
         )       ,							(
             __magic_name__      
         )       ,							(
             __magic_name__      
         )       ,							      
         )			:		Dict        =     config_and_inputs
         __magic_name__			:		List[str]        =     {'''input_ids''': input_ids, '''attention_mask''': input_mask}
         return config, inputs_dict

@require_torch
class        _UpperCamelCase	(						_UpperCAmelCase   ,				_UpperCAmelCase   ,				unittest.TestCase	):


  """simple docstring"""





  snake_case_        =    (
      (
          ErnieMModel,
          ErnieMForMultipleChoice,
          ErnieMForQuestionAnswering,
          ErnieMForSequenceClassification,
          ErnieMForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  snake_case_        =    ()
  snake_case_        =    (
      {
          """feature-extraction""": ErnieMModel,
          """question-answering""": ErnieMForQuestionAnswering,
          """text-classification""": ErnieMForSequenceClassification,
          """token-classification""": ErnieMForTokenClassification,
          """zero-shot""": ErnieMForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  snake_case_        =    False



  def 							_UpperCAmelCase      ( self					: List[str]       ,       snake_case					: Tuple       ,       snake_case					: Optional[int]       ,       snake_case					: Optional[Any]       ,       snake_case					: Tuple       ,       snake_case					: str      )	->			Any:


         '''simple docstring'''





         if pipeline_test_casse_name == "QAPipelineTests":
                return True

         return False



  def 							_UpperCAmelCase      ( self					: str      )	->			List[str]:


         '''simple docstring'''





         __magic_name__			:		str        =     ErnieMModelTester(self      )
         __magic_name__			:		List[Any]        =     ConfigTester(self       ,       config_class=snake_case       ,       hidden_size=37      )



  def 							_UpperCAmelCase      ( self					: Optional[int]      )	->			Tuple:


         '''simple docstring'''





         self.config_tester.run_common_tests()



  def 							_UpperCAmelCase      ( self					: List[Any]      )	->			Tuple:


         '''simple docstring'''





         __magic_name__			:		Tuple        =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*snake_case      )



  def 							_UpperCAmelCase      ( self					: str      )	->			Dict:


         '''simple docstring'''





         __magic_name__			:		Any        =     self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
                __magic_name__			:		Any        =     type
                self.model_tester.create_and_check_model(*snake_case      )



  def 							_UpperCAmelCase      ( self					: Dict      )	->			Optional[Any]:


         '''simple docstring'''





         __magic_name__			:		Optional[int]        =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*snake_case      )



  def 							_UpperCAmelCase      ( self					: Union[str, Any]      )	->			int:


         '''simple docstring'''





         __magic_name__			:		Any        =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*snake_case      )



  def 							_UpperCAmelCase      ( self					: Any      )	->			Optional[int]:


         '''simple docstring'''





         __magic_name__			:		Optional[Any]        =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*snake_case      )



  def 							_UpperCAmelCase      ( self					: Any      )	->			List[str]:


         '''simple docstring'''





         __magic_name__			:		Optional[int]        =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_information_extraction(*snake_case      )



  def 							_UpperCAmelCase      ( self					: str      )	->			str:


         '''simple docstring'''





         __magic_name__			:		int        =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*snake_case      )




  @slow
  def 							_UpperCAmelCase      ( self					: Any      )	->			Union[str, Any]:


         '''simple docstring'''





         for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                __magic_name__			:		Any        =     ErnieMModel.from_pretrained(snake_case      )
                self.assertIsNotNone(snake_case      )

@require_torch
class        _UpperCamelCase	(						unittest.TestCase	):


  """simple docstring"""





  @slow
  def 							_UpperCAmelCase      ( self					: Optional[Any]      )	->			Tuple:


         '''simple docstring'''





         __magic_name__			:		Union[str, Any]        =     ErnieMModel.from_pretrained('''susnato/ernie-m-base_pytorch'''      )
         model.eval()
         __magic_name__			:		Any        =     torch.tensor([[0, 1, 2, 3, 4, 5]]      )
         __magic_name__			:		str        =     model(snake_case      )[0]

         # TODO Replace vocab size
         __magic_name__			:		Optional[Any]        =     768

         __magic_name__			:		int        =     torch.Size((1, 6, hidden_size)      )
         self.assertEqual(output.shape       ,       snake_case      )

         __magic_name__			:		Tuple        =     torch.tensor(
             [[[-0.0012, 0.1245, -0.0214], [-0.0742, 0.0244, -0.0771], [-0.0333, 0.1164, -0.1554]]]      )

         self.assertTrue(torch.allclose(output[:, :3, :3]       ,       snake_case       ,       atol=1e-3      )      )





import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation


UpperCamelCase__			=    logging.get_logger(__name__)

UpperCamelCase__			=    {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__			=    {
    "vocab_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"
        ),
    },
}

UpperCamelCase__			=    {
    "microsoft/deberta-base": 5_12,
    "microsoft/deberta-large": 5_12,
    "microsoft/deberta-xlarge": 5_12,
    "microsoft/deberta-base-mnli": 5_12,
    "microsoft/deberta-large-mnli": 5_12,
    "microsoft/deberta-xlarge-mnli": 5_12,
}

UpperCamelCase__			=    {
    "microsoft/deberta-base": {"do_lower_case": False},
    "microsoft/deberta-large": {"do_lower_case": False},
}
class       _lowerCAmelCase (			_UpperCAmelCase      ):






			"""simple docstring"""





			lowercase__   :	Dict  			=      VOCAB_FILES_NAMES
			lowercase__   :	Dict  			=      PRETRAINED_VOCAB_FILES_MAP
			lowercase__   :	str  			=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase__   :	Tuple  			=      ["""input_ids""", """attention_mask""", """token_type_ids"""]
			lowercase__   :	List[Any]  			=      DebertaTokenizer





			def __init__(					self						:							Dict      ,  lowercase						:							Union[str, Any]=None      ,  lowercase						:							Union[str, Any]=None      ,  lowercase						:							Any=None      ,  lowercase						:							Optional[Any]="replace"      ,  lowercase						:							Optional[Any]="[CLS]"      ,  lowercase						:							List[Any]="[SEP]"      ,  lowercase						:							int="[SEP]"      ,  lowercase						:							str="[CLS]"      ,  lowercase						:							Any="[UNK]"      ,  lowercase						:							List[Any]="[PAD]"      ,  lowercase						:							List[str]="[MASK]"      ,  lowercase						:							Tuple=False      ,  **lowercase						:							Tuple      ,  )    ->       List[Any]:
									"""simple docstring"""







									super().__init__(
									    lowercase      ,  lowercase      ,  tokenizer_file=lowercase      ,  errors=lowercase      ,  bos_token=lowercase      ,  eos_token=lowercase      ,  unk_token=lowercase      ,  sep_token=lowercase      ,  cls_token=lowercase      ,  pad_token=lowercase      ,  mask_token=lowercase      ,  add_prefix_space=lowercase      ,  **lowercase      ,  )
									__lowercase											=					kwargs.pop("""add_bos_token"""      ,  lowercase )

									__lowercase											=					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
									if pre_tok_state.get("""add_prefix_space"""      ,  lowercase ) != add_prefix_space:
															__lowercase											=					getattr(lowercase      ,  pre_tok_state.pop("""type""" ) )
															__lowercase											=					add_prefix_space
															__lowercase											=					pre_tok_class(**lowercase )

									__lowercase											=					add_prefix_space





			@property
			def 		snake_case__      (					self						:							Tuple )    ->       str:
									"""simple docstring"""







									if self._mask_token is None:
															if self.verbose:
																					logger.error("""Using mask_token, but it is not set yet.""" )
															return None
									return str(self._mask_token )





			@mask_token.setter
			def 		snake_case__      (					self						:							List[Any]      ,  lowercase						:							Tuple )    ->       Optional[int]:
									"""simple docstring"""







									__lowercase											=					AddedToken(lowercase      ,  lstrip=lowercase      ,  rstrip=lowercase ) if isinstance(lowercase      ,  lowercase ) else value
									__lowercase											=					value





			def 		snake_case__      (					self						:							List[str]      ,  lowercase						:							List[int]      ,  lowercase						:							Optional[List[int]] = None )    ->       List[int]:
									"""simple docstring"""







									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									__lowercase											=					[self.cls_token_id]
									__lowercase											=					[self.sep_token_id]
									return cls + token_ids_a + sep + token_ids_a + sep





			def 		snake_case__      (					self						:							Dict      ,  lowercase						:							List[int]      ,  lowercase						:							Optional[List[int]] = None )    ->       List[int]:
									"""simple docstring"""







									__lowercase											=					[self.sep_token_id]
									__lowercase											=					[self.cls_token_id]

									if token_ids_a is None:
															return len(cls + token_ids_a + sep ) * [0]
									return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]





			def 		snake_case__      (					self						:							Optional[int]      ,  *lowercase						:							List[str]      ,  **lowercase						:							Tuple )    ->       BatchEncoding:
									"""simple docstring"""







									__lowercase											=					kwargs.get("""is_split_into_words"""      ,  lowercase )
									assert self.add_prefix_space or not is_split_into_words, (
									    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
									    "to use it with pretokenized inputs."
									)

									return super()._batch_encode_plus(*lowercase      ,  **lowercase )





			def 		snake_case__      (					self						:							List[Any]      ,  *lowercase						:							str      ,  **lowercase						:							List[Any] )    ->       BatchEncoding:
									"""simple docstring"""







									__lowercase											=					kwargs.get("""is_split_into_words"""      ,  lowercase )

									assert self.add_prefix_space or not is_split_into_words, (
									    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
									    "to use it with pretokenized inputs."
									)

									return super()._encode_plus(*lowercase      ,  **lowercase )





			def 		snake_case__      (					self						:							Union[str, Any]      ,  lowercase						:							str      ,  lowercase						:							Optional[str] = None )    ->       Tuple[str]:
									"""simple docstring"""







									__lowercase											=					self._tokenizer.model.save(lowercase      ,  name=lowercase )
									return tuple(lowercase )







			def 		snake_case__      (					self						:							List[str]      ,  lowercase						:							"Conversation" )    ->       List[int]:
									"""simple docstring"""







									__lowercase											=					[]
									for is_user, text in conversation.iter_texts():
															input_ids.extend(self.encode(lowercase      ,  add_special_tokens=lowercase ) + [self.eos_token_id] )

									if len(lowercase ) > self.model_max_length:
															__lowercase											=					input_ids[-self.model_max_length :]
									return input_ids





