


'''simple docstring'''


import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


UpperCAmelCase_	:     str							     =	logging.get_logger(__name__)

UpperCAmelCase_	:     List[Any]							     =	{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_	:     Optional[int]							     =	{
    'vocab_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'
        ),
    },
}

UpperCAmelCase_	:     List[str]							     =	{
    'microsoft/deberta-base': 512,
    'microsoft/deberta-large': 512,
    'microsoft/deberta-xlarge': 512,
    'microsoft/deberta-base-mnli': 512,
    'microsoft/deberta-large-mnli': 512,
    'microsoft/deberta-xlarge-mnli': 512,
}

UpperCAmelCase_	:     str							     =	{
    'microsoft/deberta-base': {'do_lower_case': False},
    'microsoft/deberta-large': {'do_lower_case': False},
}





class 					lowercase__						(		_snake_case      ):





							'''simple docstring'''


							A_							:   str   			=						VOCAB_FILES_NAMES
							A_							:   List[Any]   			=						PRETRAINED_VOCAB_FILES_MAP
							A_							:   int   			=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A_							:   str   			=						["""input_ids""", """attention_mask""", """token_type_ids"""]
							A_							:   int   			=						DebertaTokenizer






							def __init__(					self					,      __snake_case=None					,      __snake_case=None					,      __snake_case=None					,      __snake_case="replace"					,      __snake_case="[CLS]"					,      __snake_case="[SEP]"					,      __snake_case="[SEP]"					,      __snake_case="[CLS]"					,      __snake_case="[UNK]"					,      __snake_case="[PAD]"					,      __snake_case="[MASK]"					,      __snake_case=False					,      **__snake_case					,      ):
											super().__init__(
											    __snake_case					,      __snake_case					,      tokenizer_file=__snake_case					,      errors=__snake_case					,      bos_token=__snake_case					,      eos_token=__snake_case					,      unk_token=__snake_case					,      sep_token=__snake_case					,      cls_token=__snake_case					,      pad_token=__snake_case					,      mask_token=__snake_case					,      add_prefix_space=__snake_case					,      **__snake_case					,      )
											_SCREAMING_SNAKE_CASE				: int  	=   kwargs.pop("""add_bos_token"""					,      __snake_case						)

											_SCREAMING_SNAKE_CASE				: Any  	=   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
											if pre_tok_state.get("""add_prefix_space"""					,      __snake_case						) != add_prefix_space:
															_SCREAMING_SNAKE_CASE				: Tuple  	=   getattr(__snake_case					,      pre_tok_state.pop("""type"""						)						)
															_SCREAMING_SNAKE_CASE				: int  	=   add_prefix_space
															_SCREAMING_SNAKE_CASE				: Tuple  	=   pre_tok_class(**__snake_case						)

											_SCREAMING_SNAKE_CASE				: int  	=   add_prefix_space






							@property
							def UpperCAmelCase_			(					self						):
											if self._mask_token is None:
															if self.verbose:
																			logger.error("""Using mask_token, but it is not set yet."""						)
															return None
											return str(self._mask_token						)






							@mask_token.setter
							def UpperCAmelCase_			(					self					,      __snake_case						):
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   AddedToken(__snake_case					,      lstrip=__snake_case					,      rstrip=__snake_case						) if isinstance(__snake_case					,      __snake_case						) else value
											_SCREAMING_SNAKE_CASE				: int  	=   value






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case = None						):
											if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											_SCREAMING_SNAKE_CASE				: Dict  	=   [self.cls_token_id]
											_SCREAMING_SNAKE_CASE				: List[Any]  	=   [self.sep_token_id]
											return cls + token_ids_a + sep + token_ids_a + sep






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case = None						):
											_SCREAMING_SNAKE_CASE				: List[Any]  	=   [self.sep_token_id]
											_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   [self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep						) * [0]
											return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






							def UpperCAmelCase_			(					self					,      *__snake_case					,      **__snake_case						):
											_SCREAMING_SNAKE_CASE				: List[str]  	=   kwargs.get("""is_split_into_words"""					,      __snake_case						)
											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*__snake_case					,      **__snake_case						)






							def UpperCAmelCase_			(					self					,      *__snake_case					,      **__snake_case						):
											_SCREAMING_SNAKE_CASE				: List[Any]  	=   kwargs.get("""is_split_into_words"""					,      __snake_case						)

											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*__snake_case					,      **__snake_case						)






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case = None						):
											_SCREAMING_SNAKE_CASE				: Tuple  	=   self._tokenizer.model.save(__snake_case					,      name=__snake_case						)
											return tuple(__snake_case						)






							def UpperCAmelCase_			(					self					,      __snake_case						):
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   []
											for is_user, text in conversation.iter_texts():
															input_ids.extend(self.encode(__snake_case					,      add_special_tokens=__snake_case						) + [self.eos_token_id]						)

											if len(__snake_case						) > self.model_max_length:
															_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   input_ids[-self.model_max_length :]
											return input_ids








'''simple docstring'''


from ..utils import DummyObject, requires_backends





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   str   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Union[str, Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Tuple   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Tuple   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   str   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Dict   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   str   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   str   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Tuple   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   int   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Tuple   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Dict   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Union[str, Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Dict   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Tuple   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Dict   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Union[str, Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)




