


"""simple docstring"""







from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


lowerCamelCase_  :  List[Any]     		=    logging.get_logger(__name__)

lowerCamelCase_  :  Union[str, Any]     		=    {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase_  :  int     		=    {
    """vocab_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """openai/clip-vit-base-patch32""": (
            """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase_  :  int     		=    {
    """openai/clip-vit-base-patch32""": 7_7,
}






class  __A							(			_SCREAMING_SNAKE_CASE     ):






 """simple docstring"""




 __lowerCAmelCase					   =    VOCAB_FILES_NAMES
 __lowerCAmelCase					   =    PRETRAINED_VOCAB_FILES_MAP
 __lowerCAmelCase					   =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowerCAmelCase					   =    ["input_ids", "attention_mask"]
 __lowerCAmelCase					   =    CLIPTokenizer


 def __init__(						self					,       __A=None					,       __A=None					,       __A=None					,       __A="<|endoftext|>"					,       __A="<|startoftext|>"					,       __A="<|endoftext|>"					,       __A="<|endoftext|>"					,       **__A					,       )   ->	List[str]:
     super().__init__(
         __A					,       __A					,       tokenizer_file=__A					,       unk_token=__A					,       bos_token=__A					,       eos_token=__A					,       pad_token=__A					,       **__A					,       )

     if not isinstance(self.backend_tokenizer.pre_tokenizer					,       pre_tokenizers.Sequence		):
         raise ValueError(
             '''The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'''
             ''' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'''
             ''' to be compatible with this version.The easiest way to do so is'''
             ''' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'''
             ''' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'''
             ''' transformers.'''		)

     self._wrap_decode_method_backend_tokenizer()


 def  SCREAMING_SNAKE_CASE				(						self		)   ->	Optional[Any]:
     a											=self.backend_tokenizer.decode

     def new_decode_method(*__A					,       **__A		):
         a											=orig_decode_method(*__A					,       **__A		)
         a											=text.replace(self.backend_tokenizer.model.end_of_word_suffix					,       ''' '''		).strip()
         return text

     a											=new_decode_method


 def  SCREAMING_SNAKE_CASE				(						self					,       __A					,       __A = None		)   ->	List[int]:
     a											=[self.bos_token_id]
     a											=[self.eos_token_id]

     if token_ids_a is None:
         return bos_token + token_ids_a + eos_token
     return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token


 def  SCREAMING_SNAKE_CASE				(						self					,       __A					,       __A = None		)   ->	List[int]:
     a											=[self.bos_token_id]
     a											=[self.eos_token_id]

     if token_ids_a is None:
         return len(bos_token + token_ids_a + eos_token		) * [0]
     return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token		) * [0]





 def  SCREAMING_SNAKE_CASE				(						self					,       __A					,       __A = None		)   ->	Tuple[str]:
     a											=self._tokenizer.model.save(__A					,       name=__A		)
     return tuple(__A		)



"""simple docstring"""







import unittest

from datasets import load_dataset

from transformers import load_tool

from .test_tools_common import ToolTesterMixin






class  __A							(			unittest.TestCase,       _SCREAMING_SNAKE_CASE     ):






 """simple docstring"""




 def  SCREAMING_SNAKE_CASE				(						self		)   ->	str:
     a											=load_tool('''document-question-answering'''		)
     self.tool.setup()
     a											=load_tool('''document-question-answering'''					,       remote=__A		)


 def  SCREAMING_SNAKE_CASE				(						self		)   ->	Optional[Any]:
     a											=load_dataset('''hf-internal-testing/example-documents'''					,       split='''test'''		)
     a											=dataset[0]['''image''']

     a											=self.tool(__A					,       '''When is the coffee break?'''		)
     self.assertEqual(__A					,       '''11-14 to 11:39 a.m.'''		)


 def  SCREAMING_SNAKE_CASE				(						self		)   ->	Optional[int]:
     a											=load_dataset('''hf-internal-testing/example-documents'''					,       split='''test'''		)
     a											=dataset[0]['''image''']

     a											=self.remote_tool(__A					,       '''When is the coffee break?'''		)
     self.assertEqual(__A					,       '''11-14 to 11:39 a.m.'''		)


 def  SCREAMING_SNAKE_CASE				(						self		)   ->	Union[str, Any]:
     a											=load_dataset('''hf-internal-testing/example-documents'''					,       split='''test'''		)
     a											=dataset[0]['''image''']

     self.tool(document=__A					,       question='''When is the coffee break?'''		)





 def  SCREAMING_SNAKE_CASE				(						self		)   ->	Optional[int]:
     a											=load_dataset('''hf-internal-testing/example-documents'''					,       split='''test'''		)
     a											=dataset[0]['''image''']

     a											=self.remote_tool(document=__A					,       question='''When is the coffee break?'''		)
     self.assertEqual(__A					,       '''11-14 to 11:39 a.m.'''		)