


import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


__snake_case             =	logging.get_logger(__name__)

__snake_case             =	{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__snake_case             =	{
    '''vocab_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'''
        ),
    },
}

__snake_case             =	{
    '''microsoft/deberta-base''': 5_1_2,
    '''microsoft/deberta-large''': 5_1_2,
    '''microsoft/deberta-xlarge''': 5_1_2,
    '''microsoft/deberta-base-mnli''': 5_1_2,
    '''microsoft/deberta-large-mnli''': 5_1_2,
    '''microsoft/deberta-xlarge-mnli''': 5_1_2,
}

__snake_case             =	{
    '''microsoft/deberta-base''': {'''do_lower_case''': False},
    '''microsoft/deberta-large''': {'''do_lower_case''': False},
}



class   __lowerCamelCase (_a	):
       _lowercase       							=    VOCAB_FILES_NAMES
       _lowercase       							=    PRETRAINED_VOCAB_FILES_MAP
       _lowercase       							=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _lowercase       							=    ["""input_ids""", """attention_mask""", """token_type_ids"""]
       _lowercase       							=    DebertaTokenizer







       def __init__( self:   Union[str, Any],A_:   int=None,A_:   Tuple=None,A_:   Any=None,A_:   Any="replace",A_:   Optional[Any]="[CLS]",A_:   Dict="[SEP]",A_:   int="[SEP]",A_:   str="[CLS]",A_:   Any="[UNK]",A_:   List[Any]="[PAD]",A_:   int="[MASK]",A_:   Tuple=False,**A_:   int,):






            '''simple docstring'''




            super().__init__(
                A_,A_,tokenizer_file=A_,errors=A_,bos_token=A_,eos_token=A_,unk_token=A_,sep_token=A_,cls_token=A_,pad_token=A_,mask_token=A_,add_prefix_space=A_,**A_,)
            __UpperCamelCase   = kwargs.pop('add_bos_token',A_				)

            __UpperCamelCase   = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
            if pre_tok_state.get('add_prefix_space',A_				) != add_prefix_space:
                 __UpperCamelCase   = getattr(A_,pre_tok_state.pop('type'				)				)
                 __UpperCamelCase   = add_prefix_space
                 __UpperCamelCase   = pre_tok_class(**A_				)

            __UpperCamelCase   = add_prefix_space







       @property
       def 		snake_case_  ( self:   Tuple				):






            '''simple docstring'''




            if self._mask_token is None:
                 if self.verbose:
                      logger.error('Using mask_token, but it is not set yet.'				)
                 return None
            return str(self._mask_token				)







       @mask_token.setter
       def 		snake_case_  ( self:   Any,A_:   List[str]				):






            '''simple docstring'''




            __UpperCamelCase   = AddedToken(A_,lstrip=A_,rstrip=A_				) if isinstance(A_,A_				) else value
            __UpperCamelCase   = value







       def 		snake_case_  ( self:   Tuple,A_:   List[int],A_:   Optional[List[int]] = None				):






            '''simple docstring'''




            if token_ids_a is None:
                 return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
            __UpperCamelCase   = [self.cls_token_id]
            __UpperCamelCase   = [self.sep_token_id]
            return cls + token_ids_a + sep + token_ids_a + sep







       def 		snake_case_  ( self:   Any,A_:   List[int],A_:   Optional[List[int]] = None				):






            '''simple docstring'''




            __UpperCamelCase   = [self.sep_token_id]
            __UpperCamelCase   = [self.cls_token_id]

            if token_ids_a is None:
                 return len(cls + token_ids_a + sep				) * [0]
            return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







       def 		snake_case_  ( self:   List[str],*A_:   int,**A_:   int				):






            '''simple docstring'''




            __UpperCamelCase   = kwargs.get('is_split_into_words',A_				)
            assert self.add_prefix_space or not is_split_into_words, (
                F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                "to use it with pretokenized inputs."
            )

            return super()._batch_encode_plus(*A_,**A_				)







       def 		snake_case_  ( self:   List[Any],*A_:   Optional[int],**A_:   int				):






            '''simple docstring'''




            __UpperCamelCase   = kwargs.get('is_split_into_words',A_				)

            assert self.add_prefix_space or not is_split_into_words, (
                F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                "to use it with pretokenized inputs."
            )

            return super()._encode_plus(*A_,**A_				)







       def 		snake_case_  ( self:   Tuple,A_:   str,A_:   Optional[str] = None				):






            '''simple docstring'''




            __UpperCamelCase   = self._tokenizer.model.save(A_,name=A_				)
            return tuple(A_				)







       def 		snake_case_  ( self:   Dict,A_:   "Conversation"				):






            '''simple docstring'''




            __UpperCamelCase   = []
            for is_user, text in conversation.iter_texts():
                 input_ids.extend(self.encode(A_,add_special_tokens=A_				) + [self.eos_token_id]				)

            if len(A_				) > self.model_max_length:
                 __UpperCamelCase   = input_ids[-self.model_max_length :]
            return input_ids






import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import DeiTImageProcessor



class   __lowerCamelCase (unittest.TestCase	):







       def __init__( self:   List[str],A_:   List[Any],A_:   List[Any]=7,A_:   str=3,A_:   Tuple=18,A_:   Union[str, Any]=30,A_:   Union[str, Any]=400,A_:   List[str]=True,A_:   List[Any]=None,A_:   Tuple=True,A_:   Union[str, Any]=None,A_:   int=True,A_:   Any=[0.5, 0.5, 0.5],A_:   Optional[Any]=[0.5, 0.5, 0.5],):






            '''simple docstring'''




            __UpperCamelCase   = size if size is not None else {'height': 20, 'width': 20}
            __UpperCamelCase   = crop_size if crop_size is not None else {'height': 18, 'width': 18}

            __UpperCamelCase   = parent
            __UpperCamelCase   = batch_size
            __UpperCamelCase   = num_channels
            __UpperCamelCase   = image_size
            __UpperCamelCase   = min_resolution
            __UpperCamelCase   = max_resolution
            __UpperCamelCase   = do_resize
            __UpperCamelCase   = size
            __UpperCamelCase   = do_center_crop
            __UpperCamelCase   = crop_size
            __UpperCamelCase   = do_normalize
            __UpperCamelCase   = image_mean
            __UpperCamelCase   = image_std







       def 		snake_case_  ( self:   List[str]				):






            '''simple docstring'''




            return {
                "do_resize": self.do_resize,
                "size": self.size,
                "do_center_crop": self.do_center_crop,
                "crop_size": self.crop_size,
                "do_normalize": self.do_normalize,
                "image_mean": self.image_mean,
                "image_std": self.image_std,
            }



@require_torch
@require_vision
class   __lowerCamelCase (_a							,			unittest.TestCase	):
       _lowercase       							=    DeiTImageProcessor if is_vision_available() else None
       _lowercase       							=    True







       def 		snake_case_  ( self:   List[Any]				):






            '''simple docstring'''




            __UpperCamelCase   = DeiTImageProcessingTester(self				)







       @property
       def 		snake_case_  ( self:   Optional[int]				):






            '''simple docstring'''




            return self.image_processor_tester.prepare_image_processor_dict()







       def 		snake_case_  ( self:   Union[str, Any]				):






            '''simple docstring'''




            __UpperCamelCase   = self.image_processing_class(**self.image_processor_dict				)
            self.assertTrue(hasattr(A_,'do_resize'				)				)
            self.assertTrue(hasattr(A_,'size'				)				)
            self.assertTrue(hasattr(A_,'do_center_crop'				)				)
            self.assertTrue(hasattr(A_,'center_crop'				)				)
            self.assertTrue(hasattr(A_,'do_normalize'				)				)
            self.assertTrue(hasattr(A_,'image_mean'				)				)
            self.assertTrue(hasattr(A_,'image_std'				)				)







       def 		snake_case_  ( self:   Optional[int]				):






            '''simple docstring'''




            __UpperCamelCase   = self.image_processing_class.from_dict(self.image_processor_dict				)
            self.assertEqual(image_processor.size,{'height': 20, 'width': 20}				)
            self.assertEqual(image_processor.crop_size,{'height': 18, 'width': 18}				)

            __UpperCamelCase   = self.image_processing_class.from_dict(self.image_processor_dict,size=42,crop_size=84				)
            self.assertEqual(image_processor.size,{'height': 42, 'width': 42}				)
            self.assertEqual(image_processor.crop_size,{'height': 84, 'width': 84}				)







       def 		snake_case_  ( self:   Tuple				):






            '''simple docstring'''




            pass







       def 		snake_case_  ( self:   Optional[int]				):






            '''simple docstring'''




            __UpperCamelCase   = self.image_processing_class(**self.image_processor_dict				)
            # create random PIL images
            __UpperCamelCase   = prepare_image_inputs(self.image_processor_tester,equal_resolution=A_				)
            for image in image_inputs:
                 self.assertIsInstance(A_,Image.Image				)

            # Test not batched input
            __UpperCamelCase   = image_processing(image_inputs[0],return_tensors='pt'				).pixel_values
            self.assertEqual(
                encoded_images.shape,(
                    1,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size['height'],
                    self.image_processor_tester.crop_size['width'],
                ),)

            # Test batched
            __UpperCamelCase   = image_processing(A_,return_tensors='pt'				).pixel_values
            self.assertEqual(
                encoded_images.shape,(
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size['height'],
                    self.image_processor_tester.crop_size['width'],
                ),)







       def 		snake_case_  ( self:   List[str]				):






            '''simple docstring'''




            __UpperCamelCase   = self.image_processing_class(**self.image_processor_dict				)
            # create random numpy tensors
            __UpperCamelCase   = prepare_image_inputs(self.image_processor_tester,equal_resolution=A_,numpify=A_				)
            for image in image_inputs:
                 self.assertIsInstance(A_,np.ndarray				)

            # Test not batched input
            __UpperCamelCase   = image_processing(image_inputs[0],return_tensors='pt'				).pixel_values
            self.assertEqual(
                encoded_images.shape,(
                    1,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size['height'],
                    self.image_processor_tester.crop_size['width'],
                ),)

            # Test batched
            __UpperCamelCase   = image_processing(A_,return_tensors='pt'				).pixel_values
            self.assertEqual(
                encoded_images.shape,(
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size['height'],
                    self.image_processor_tester.crop_size['width'],
                ),)







       def 		snake_case_  ( self:   Any				):






            '''simple docstring'''




            __UpperCamelCase   = self.image_processing_class(**self.image_processor_dict				)
            # create random PyTorch tensors
            __UpperCamelCase   = prepare_image_inputs(self.image_processor_tester,equal_resolution=A_,torchify=A_				)
            for image in image_inputs:
                 self.assertIsInstance(A_,torch.Tensor				)

            # Test not batched input
            __UpperCamelCase   = image_processing(image_inputs[0],return_tensors='pt'				).pixel_values
            self.assertEqual(
                encoded_images.shape,(
                    1,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size['height'],
                    self.image_processor_tester.crop_size['width'],
                ),)

            # Test batched
            __UpperCamelCase   = image_processing(A_,return_tensors='pt'				).pixel_values
            self.assertEqual(
                encoded_images.shape,(
                    self.image_processor_tester.batch_size,
                    self.image_processor_tester.num_channels,
                    self.image_processor_tester.crop_size['height'],
                    self.image_processor_tester.crop_size['width'],
                ),)


