




'''simple docstring'''





import unittest

from transformers import ErnieMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           ErnieMForInformationExtraction,
           ErnieMForMultipleChoice,
           ErnieMForQuestionAnswering,
           ErnieMForSequenceClassification,
           ErnieMForTokenClassification,
           ErnieMModel,
       )
       from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST






class 						lowercase     :






      def __init__(self ,  a__ ,  a__=1_3 ,  a__=7 ,  a__=True ,  a__=True ,  a__=True ,  a__=9_9 ,  a__=3_2 ,  a__=5 ,  a__=4 ,  a__=3_7 ,  a__="gelu" ,  a__=0.1 ,  a__=0.1 ,  a__=5_1_2 ,  a__=1_6 ,  a__=2 ,  a__=0.02 ,  a__=3 ,  a__=4 ,  a__=None ,  )       ->   int:
         SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= parent
         SCREAMING_SNAKE_CASE__			:				str       						= batch_size
         SCREAMING_SNAKE_CASE__			:				str       						= seq_length
         SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= is_training
         SCREAMING_SNAKE_CASE__			:				Any       						= use_input_mask
         SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= use_labels
         SCREAMING_SNAKE_CASE__			:				str       						= vocab_size
         SCREAMING_SNAKE_CASE__			:				int       						= hidden_size
         SCREAMING_SNAKE_CASE__			:				Tuple       						= num_hidden_layers
         SCREAMING_SNAKE_CASE__			:				int       						= num_attention_heads
         SCREAMING_SNAKE_CASE__			:				int       						= intermediate_size
         SCREAMING_SNAKE_CASE__			:				List[Any]       						= hidden_act
         SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= hidden_dropout_prob
         SCREAMING_SNAKE_CASE__			:				Tuple       						= attention_probs_dropout_prob
         SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= max_position_embeddings
         SCREAMING_SNAKE_CASE__			:				str       						= type_vocab_size
         SCREAMING_SNAKE_CASE__			:				Tuple       						= type_sequence_label_size
         SCREAMING_SNAKE_CASE__			:				List[Any]       						= initializer_range
         SCREAMING_SNAKE_CASE__			:				int       						= num_labels
         SCREAMING_SNAKE_CASE__			:				List[str]       						= num_choices
         SCREAMING_SNAKE_CASE__			:				Optional[int]       						= scope






      def    A__(self					)       ->   Tuple:
         SCREAMING_SNAKE_CASE__			:				Any       						= ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size					)

         SCREAMING_SNAKE_CASE__			:				int       						= None
         if self.use_input_mask:
            SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= random_attention_mask([self.batch_size, self.seq_length]					)

         SCREAMING_SNAKE_CASE__			:				str       						= None
         SCREAMING_SNAKE_CASE__			:				int       						= None
         SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= None
         if self.use_labels:
            SCREAMING_SNAKE_CASE__			:				Optional[int]       						= ids_tensor([self.batch_size] ,  self.type_sequence_label_size					)
            SCREAMING_SNAKE_CASE__			:				str       						= ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels					)
            SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= ids_tensor([self.batch_size] ,  self.num_choices					)

         SCREAMING_SNAKE_CASE__			:				Dict       						= self.get_config()

         return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






      def    A__(self					)       ->   Dict:
         SCREAMING_SNAKE_CASE__			:				List[Any]       						= ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size					)

         SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= None
         if self.use_input_mask:
            SCREAMING_SNAKE_CASE__			:				Optional[int]       						= random_attention_mask([self.batch_size, self.seq_length]					)
         SCREAMING_SNAKE_CASE__			:				Tuple       						= self.get_config()

         return config, input_ids, input_mask






      def    A__(self					)       ->   Union[str, Any]:
         return ErnieMConfig(
             vocab_size=self.vocab_size ,  hidden_size=self.hidden_size ,  num_hidden_layers=self.num_hidden_layers ,  num_attention_heads=self.num_attention_heads ,  intermediate_size=self.intermediate_size ,  hidden_act=self.hidden_act ,  hidden_dropout_prob=self.hidden_dropout_prob ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob ,  max_position_embeddings=self.max_position_embeddings ,  type_vocab_size=self.type_vocab_size ,  initializer_range=self.initializer_range ,  )






      def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   Union[str, Any]:
         SCREAMING_SNAKE_CASE__			:				str       						= ErnieMModel(config=a__					)
         model.to(a__					)
         model.eval()
         SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= model(a__ ,  return_dict=a__					)
         self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)					)






      def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   Dict:
         SCREAMING_SNAKE_CASE__			:				Optional[int]       						= ErnieMForQuestionAnswering(config=a__					)
         model.to(a__					)
         model.eval()
         SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= model(
             a__ ,  attention_mask=a__ ,  start_positions=a__ ,  end_positions=a__ ,  )
         self.parent.assertEqual(result.start_logits.shape ,  (self.batch_size, self.seq_length)					)
         self.parent.assertEqual(result.end_logits.shape ,  (self.batch_size, self.seq_length)					)






      def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   str:
         SCREAMING_SNAKE_CASE__			:				Tuple       						= ErnieMForInformationExtraction(config=a__					)
         model.to(a__					)
         model.eval()
         SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= torch.ones_like(a__ ,  dtype=torch.floataa					)
         SCREAMING_SNAKE_CASE__			:				Any       						= model(
             a__ ,  attention_mask=a__ ,  start_positions=a__ ,  end_positions=a__ ,  )
         self.parent.assertEqual(result.start_logits.shape ,  (self.batch_size, self.seq_length)					)
         self.parent.assertEqual(result.end_logits.shape ,  (self.batch_size, self.seq_length)					)






      def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   int:
         SCREAMING_SNAKE_CASE__			:				Optional[int]       						= self.num_labels
         SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= ErnieMForSequenceClassification(a__					)
         model.to(a__					)
         model.eval()
         SCREAMING_SNAKE_CASE__			:				Dict       						= model(a__ ,  attention_mask=a__ ,  labels=a__					)
         self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_labels)					)






      def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   str:
         SCREAMING_SNAKE_CASE__			:				int       						= self.num_labels
         SCREAMING_SNAKE_CASE__			:				str       						= ErnieMForTokenClassification(config=a__					)
         model.to(a__					)
         model.eval()
         input_ids.to(a__					)
         input_mask.to(a__					)
         token_labels.to(a__					)

         SCREAMING_SNAKE_CASE__			:				Optional[int]       						= model(a__ ,  attention_mask=a__ ,  labels=a__					)

         self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.num_labels)					)






      def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   Optional[int]:
         SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= self.num_choices
         SCREAMING_SNAKE_CASE__			:				str       						= ErnieMForMultipleChoice(config=a__					)
         model.to(a__					)
         model.eval()
         SCREAMING_SNAKE_CASE__			:				Dict       						= input_ids.unsqueeze(1					).expand(-1 ,  self.num_choices ,  -1					).contiguous()
         SCREAMING_SNAKE_CASE__			:				int       						= input_mask.unsqueeze(1					).expand(-1 ,  self.num_choices ,  -1					).contiguous()
         SCREAMING_SNAKE_CASE__			:				List[Any]       						= model(
             a__ ,  attention_mask=a__ ,  labels=a__ ,  )
         self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_choices)					)






      def    A__(self					)       ->   int:
         SCREAMING_SNAKE_CASE__			:				Optional[int]       						= self.prepare_config_and_inputs()
         (
             (
             SCREAMING_SNAKE_CASE__
         )						,				(
             SCREAMING_SNAKE_CASE__
         )						,				(
             SCREAMING_SNAKE_CASE__
         )						,				(
             SCREAMING_SNAKE_CASE__
         )						,				(
             SCREAMING_SNAKE_CASE__
         )						,				(
             SCREAMING_SNAKE_CASE__
         )						,				
         )			:				Any       						= config_and_inputs
         SCREAMING_SNAKE_CASE__			:				Any       						= {'''input_ids''': input_ids, '''attention_mask''': input_mask}
         return config, inputs_dict






@require_torch
class 						lowercase     (   _a				,			_a				,			unittest.TestCase):
      __lowercase 		=  (
          (
              ErnieMModel,
              ErnieMForMultipleChoice,
              ErnieMForQuestionAnswering,
              ErnieMForSequenceClassification,
              ErnieMForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      __lowercase 		=  ()
      __lowercase 		=  (
          {
              'feature-extraction': ErnieMModel,
              'question-answering': ErnieMForQuestionAnswering,
              'text-classification': ErnieMForSequenceClassification,
              'token-classification': ErnieMForTokenClassification,
              'zero-shot': ErnieMForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      __lowercase 		=  False






      def    A__(self ,  a__ ,  a__ ,  a__ ,  a__ ,  a__					)       ->   Optional[Any]:
         if pipeline_test_casse_name == "QAPipelineTests":
            return True

         return False






      def    A__(self					)       ->   Dict:
         SCREAMING_SNAKE_CASE__			:				List[Any]       						= ErnieMModelTester(self					)
         SCREAMING_SNAKE_CASE__			:				int       						= ConfigTester(self ,  config_class=a__ ,  hidden_size=3_7					)






      def    A__(self					)       ->   int:
         self.config_tester.run_common_tests()






      def    A__(self					)       ->   Optional[int]:
         SCREAMING_SNAKE_CASE__			:				Optional[int]       						= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*a__					)






      def    A__(self					)       ->   Dict:
         SCREAMING_SNAKE_CASE__			:				List[Any]       						= self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
            SCREAMING_SNAKE_CASE__			:				int       						= type
            self.model_tester.create_and_check_model(*a__					)






      def    A__(self					)       ->   Dict:
         SCREAMING_SNAKE_CASE__			:				Tuple       						= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*a__					)






      def    A__(self					)       ->   Any:
         SCREAMING_SNAKE_CASE__			:				List[str]       						= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*a__					)






      def    A__(self					)       ->   Optional[Any]:
         SCREAMING_SNAKE_CASE__			:				str       						= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*a__					)






      def    A__(self					)       ->   List[str]:
         SCREAMING_SNAKE_CASE__			:				str       						= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_information_extraction(*a__					)






      def    A__(self					)       ->   List[Any]:
         SCREAMING_SNAKE_CASE__			:				List[str]       						= self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*a__					)






      @slow
      def    A__(self					)       ->   str:
         for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            SCREAMING_SNAKE_CASE__			:				Any       						= ErnieMModel.from_pretrained(a__					)
            self.assertIsNotNone(a__					)






@require_torch
class 						lowercase     (   unittest.TestCase):






      @slow
      def    A__(self					)       ->   Union[str, Any]:
         SCREAMING_SNAKE_CASE__			:				Tuple       						= ErnieMModel.from_pretrained('''susnato/ernie-m-base_pytorch'''					)
         model.eval()
         SCREAMING_SNAKE_CASE__			:				List[str]       						= torch.tensor([[0, 1, 2, 3, 4, 5]]					)
         SCREAMING_SNAKE_CASE__			:				Optional[int]       						= model(a__					)[0]

         # TODO Replace vocab size
         SCREAMING_SNAKE_CASE__			:				List[str]       						= 7_6_8

         SCREAMING_SNAKE_CASE__			:				List[Any]       						= torch.Size((1, 6, hidden_size)					)
         self.assertEqual(output.shape ,  a__					)

         SCREAMING_SNAKE_CASE__			:				List[str]       						= torch.tensor(
             [[[-0.00_12, 0.12_45, -0.02_14], [-0.07_42, 0.02_44, -0.07_71], [-0.03_33, 0.11_64, -0.15_54]]]					)

         self.assertTrue(torch.allclose(output[:, :3, :3] ,  a__ ,  atol=1E-3					)					)







'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowercase__			=	logging.get_logger(__name__)

lowercase__			=	{'''vocab_file''': '''vocab.txt'''}

lowercase__			=	{
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

lowercase__			=	{
    '''tau/splinter-base''': 5_1_2,
    '''tau/splinter-base-qass''': 5_1_2,
    '''tau/splinter-large''': 5_1_2,
    '''tau/splinter-large-qass''': 5_1_2,
}

lowercase__			=	{
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}






class 						lowercase     (   UpperCAmelCase_):
      __lowercase 		=  VOCAB_FILES_NAMES
      __lowercase 		=  PRETRAINED_VOCAB_FILES_MAP
      __lowercase 		=  PRETRAINED_INIT_CONFIGURATION
      __lowercase 		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __lowercase 		=  SplinterTokenizer






      def __init__(self ,  a__=None ,  a__=None ,  a__=True ,  a__="[UNK]" ,  a__="[SEP]" ,  a__="[PAD]" ,  a__="[CLS]" ,  a__="[MASK]" ,  a__="[QUESTION]" ,  a__=True ,  a__=None ,  **a__ ,  )       ->   Any:
         super().__init__(
             a__ ,  tokenizer_file=a__ ,  do_lower_case=a__ ,  unk_token=a__ ,  sep_token=a__ ,  pad_token=a__ ,  cls_token=a__ ,  mask_token=a__ ,  tokenize_chinese_chars=a__ ,  strip_accents=a__ ,  additional_special_tokens=(question_token,) ,  **a__ ,  )

         SCREAMING_SNAKE_CASE__			:				Tuple       						= json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
         if (
             pre_tok_state.get('''lowercase''' ,  a__					) != do_lower_case
             or pre_tok_state.get('''strip_accents''' ,  a__					) != strip_accents
         ):
            SCREAMING_SNAKE_CASE__			:				List[Any]       						= getattr(a__ ,  pre_tok_state.pop('''type'''					)					)
            SCREAMING_SNAKE_CASE__			:				Optional[int]       						= do_lower_case
            SCREAMING_SNAKE_CASE__			:				Optional[int]       						= strip_accents
            SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= pre_tok_class(**a__					)

         SCREAMING_SNAKE_CASE__			:				Optional[int]       						= do_lower_case






      @property
      def    A__(self					)       ->   List[str]:
         return self.convert_tokens_to_ids(self.question_token					)






      def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         SCREAMING_SNAKE_CASE__			:				Dict       						= [self.cls_token_id]
         SCREAMING_SNAKE_CASE__			:				Tuple       						= [self.sep_token_id]
         SCREAMING_SNAKE_CASE__			:				List[Any]       						= [self.question_token_id] + [self.convert_tokens_to_ids('''.'''					)]
         if self.padding_side == "right":
            # Input is question-then-context
            return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
         else:
            # Input is context-then-question
            return cls + token_ids_a + sep + token_ids_a + question_suffix + sep






      def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
         SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= [self.sep_token_id]
         SCREAMING_SNAKE_CASE__			:				Any       						= [self.cls_token_id]
         SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= [self.question_token_id] + [self.convert_tokens_to_ids('''.'''					)]
         if token_ids_a is None:
            return len(cls + token_ids_a + sep					) * [0]

         if self.padding_side == "right":
            # Input is question-then-context
            return len(cls + token_ids_a + question_suffix + sep					) * [0] + len(token_ids_a + sep					) * [1]
         else:
            # Input is context-then-question
            return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + question_suffix + sep					) * [1]






      def    A__(self ,  a__ ,  a__ = None					)       ->   Tuple[str]:
         SCREAMING_SNAKE_CASE__			:				int       						= self._tokenizer.model.save(a__ ,  name=a__					)
         return tuple(a__					)

