import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


A_:      List[str]        =			logging.get_logger(__name__)

A_:      Any        =			{"""vocab_file""": """vocab.txt"""}

A_:      Optional[int]        =			{
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

A_:      str        =			{
    """tau/splinter-base""": 512,
    """tau/splinter-base-qass""": 512,
    """tau/splinter-large""": 512,
    """tau/splinter-large-qass""": 512,
}

A_:      Optional[Any]        =			{
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}


class _lowercase					(       a_						):





							"""simple docstring"""




							lowerCAmelCase__       =					VOCAB_FILES_NAMES
							lowerCAmelCase__       =					PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase__       =					PRETRAINED_INIT_CONFIGURATION
							lowerCAmelCase__       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase__       =					SplinterTokenizer



							def __init__(    self     ,   UpperCAmelCase=None     ,   UpperCAmelCase=None     ,   UpperCAmelCase=True     ,   UpperCAmelCase="[UNK]"     ,   UpperCAmelCase="[SEP]"     ,   UpperCAmelCase="[PAD]"     ,   UpperCAmelCase="[CLS]"     ,   UpperCAmelCase="[MASK]"     ,   UpperCAmelCase="[QUESTION]"     ,   UpperCAmelCase=True     ,   UpperCAmelCase=None     ,   **UpperCAmelCase     ,   ):







														'''simple docstring'''






														super().__init__(
														    _snake_case     ,   tokenizer_file=_snake_case     ,   do_lower_case=_snake_case     ,   unk_token=_snake_case     ,   sep_token=_snake_case     ,   pad_token=_snake_case     ,   cls_token=_snake_case     ,   mask_token=_snake_case     ,   tokenize_chinese_chars=_snake_case     ,   strip_accents=_snake_case     ,   additional_special_tokens=(question_token,)     ,   **_snake_case     ,   )

														_lowercase				=	json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
														if (
														    pre_tok_state.get("""lowercase"""     ,   _snake_case  ) != do_lower_case
														    or pre_tok_state.get("""strip_accents"""     ,   _snake_case  ) != strip_accents
														):
																					_lowercase				=	getattr(_snake_case     ,   pre_tok_state.pop("""type"""  )  )
																					_lowercase				=	do_lower_case
																					_lowercase				=	strip_accents
																					_lowercase				=	pre_tok_class(**_snake_case  )

														_lowercase				=	do_lower_case



							@property
							def     _UpperCAmelCase     (    self  ):







														'''simple docstring'''






														return self.convert_tokens_to_ids(self.question_token  )



							def     _UpperCAmelCase     (    self     ,   UpperCAmelCase     ,   UpperCAmelCase = None  ):







														'''simple docstring'''






														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														_lowercase				=	[self.cls_token_id]
														_lowercase				=	[self.sep_token_id]
														_lowercase				=	[self.question_token_id] + [self.convert_tokens_to_ids("""."""  )]
														if self.padding_side == "right":
																					# Input is question-then-context
																					return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
														else:
																					# Input is context-then-question
																					return cls + token_ids_a + sep + token_ids_a + question_suffix + sep



							def     _UpperCAmelCase     (    self     ,   UpperCAmelCase     ,   UpperCAmelCase = None  ):







														'''simple docstring'''






														_lowercase				=	[self.sep_token_id]
														_lowercase				=	[self.cls_token_id]
														_lowercase				=	[self.question_token_id] + [self.convert_tokens_to_ids("""."""  )]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep  ) * [0]

														if self.padding_side == "right":
																					# Input is question-then-context
																					return len(cls + token_ids_a + question_suffix + sep  ) * [0] + len(token_ids_a + sep  ) * [1]
														else:
																					# Input is context-then-question
																					return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + question_suffix + sep  ) * [1]



							def     _UpperCAmelCase     (    self     ,   UpperCAmelCase     ,   UpperCAmelCase = None  ):







														'''simple docstring'''






														_lowercase				=	self._tokenizer.model.save(_snake_case     ,   name=_snake_case  )
														return tuple(_snake_case  )



import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_vision
from transformers.utils import is_vision_available


if is_vision_available():
				from PIL import Image

				from transformers import AutoProcessor, BertTokenizer, CLIPImageProcessor, GitProcessor, PreTrainedTokenizerFast


@require_vision
class _lowercase					(       unittest.TestCase						):





							"""simple docstring"""




							def     _UpperCAmelCase       (    self  ):







														'''simple docstring'''






														_lowercase				=	tempfile.mkdtemp()

														_lowercase				=	CLIPImageProcessor()
														_lowercase				=	BertTokenizer.from_pretrained(
														    """hf-internal-testing/tiny-random-BertModel"""     ,   model_input_names=["""input_ids""", """attention_mask"""]  )

														_lowercase				=	GitProcessor(UpperCAmelCase     ,   UpperCAmelCase  )

														processor.save_pretrained(self.tmpdirname  )



							def     _UpperCAmelCase       (    self     ,   **UpperCAmelCase  ):







														'''simple docstring'''






														return AutoProcessor.from_pretrained(self.tmpdirname     ,   **UpperCAmelCase  ).tokenizer



							def     _UpperCAmelCase       (    self     ,   **UpperCAmelCase  ):







														'''simple docstring'''






														return AutoProcessor.from_pretrained(self.tmpdirname     ,   **UpperCAmelCase  ).image_processor



							def     _UpperCAmelCase       (    self  ):







														'''simple docstring'''






														shutil.rmtree(self.tmpdirname  )



							def     _UpperCAmelCase       (    self  ):







														'''simple docstring'''






														_lowercase				=	[np.random.randint(255     ,   size=(3, 30, 400)     ,   dtype=np.uinta  )]

														_lowercase				=	[Image.fromarray(np.moveaxis(UpperCAmelCase     ,   0     ,   -1  )  ) for x in image_inputs]

														return image_inputs



							def     _UpperCAmelCase       (    self  ):







														'''simple docstring'''






														_lowercase				=	GitProcessor(tokenizer=self.get_tokenizer()     ,   image_processor=self.get_image_processor()  )
														processor.save_pretrained(self.tmpdirname  )

														_lowercase				=	self.get_tokenizer(bos_token="""(BOS)"""     ,   eos_token="""(EOS)"""  )
														_lowercase				=	self.get_image_processor(do_normalize=UpperCAmelCase     ,   padding_value=1.0  )

														_lowercase				=	GitProcessor.from_pretrained(
														    self.tmpdirname     ,   bos_token="""(BOS)"""     ,   eos_token="""(EOS)"""     ,   do_normalize=UpperCAmelCase     ,   padding_value=1.0  )

														self.assertEqual(processor.tokenizer.get_vocab()     ,   tokenizer_add_kwargs.get_vocab()  )
														self.assertIsInstance(processor.tokenizer     ,   UpperCAmelCase  )

														self.assertEqual(processor.image_processor.to_json_string()     ,   image_processor_add_kwargs.to_json_string()  )
														self.assertIsInstance(processor.image_processor     ,   UpperCAmelCase  )



							def     _UpperCAmelCase       (    self  ):







														'''simple docstring'''






														_lowercase				=	self.get_image_processor()
														_lowercase				=	self.get_tokenizer()

														_lowercase				=	GitProcessor(tokenizer=UpperCAmelCase     ,   image_processor=UpperCAmelCase  )

														_lowercase				=	self.prepare_image_inputs()

														_lowercase				=	image_processor(UpperCAmelCase     ,   return_tensors="""np"""  )
														_lowercase				=	processor(images=UpperCAmelCase     ,   return_tensors="""np"""  )

														for key in input_feat_extract.keys():
																					self.assertAlmostEqual(input_feat_extract[key].sum()     ,   input_processor[key].sum()     ,   delta=1e-2  )



							def     _UpperCAmelCase       (    self  ):







														'''simple docstring'''






														_lowercase				=	self.get_image_processor()
														_lowercase				=	self.get_tokenizer()

														_lowercase				=	GitProcessor(tokenizer=UpperCAmelCase     ,   image_processor=UpperCAmelCase  )

														_lowercase				=	"""lower newer"""

														_lowercase				=	processor(text=UpperCAmelCase  )

														_lowercase				=	tokenizer(UpperCAmelCase     ,   return_token_type_ids=UpperCAmelCase  )

														for key in encoded_tok.keys():
																					self.assertListEqual(encoded_tok[key]     ,   encoded_processor[key]  )



							def     _UpperCAmelCase       (    self  ):







														'''simple docstring'''






														_lowercase				=	self.get_image_processor()
														_lowercase				=	self.get_tokenizer()

														_lowercase				=	GitProcessor(tokenizer=UpperCAmelCase     ,   image_processor=UpperCAmelCase  )

														_lowercase				=	"""lower newer"""
														_lowercase				=	self.prepare_image_inputs()

														_lowercase				=	processor(text=UpperCAmelCase     ,   images=UpperCAmelCase  )

														self.assertListEqual(list(inputs.keys()  )     ,   ["""input_ids""", """attention_mask""", """pixel_values"""]  )

														# test if it raises when no input is passed
														with pytest.raises(UpperCAmelCase  ):
																					processor()



							def     _UpperCAmelCase       (    self  ):







														'''simple docstring'''






														_lowercase				=	self.get_image_processor()
														_lowercase				=	self.get_tokenizer()

														_lowercase				=	GitProcessor(tokenizer=UpperCAmelCase     ,   image_processor=UpperCAmelCase  )

														_lowercase				=	[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

														_lowercase				=	processor.batch_decode(UpperCAmelCase  )
														_lowercase				=	tokenizer.batch_decode(UpperCAmelCase  )

														self.assertListEqual(UpperCAmelCase     ,   UpperCAmelCase  )



							def     _UpperCAmelCase       (    self  ):







														'''simple docstring'''






														_lowercase				=	self.get_image_processor()
														_lowercase				=	self.get_tokenizer()

														_lowercase				=	GitProcessor(tokenizer=UpperCAmelCase     ,   image_processor=UpperCAmelCase  )

														_lowercase				=	"""lower newer"""
														_lowercase				=	self.prepare_image_inputs()

														_lowercase				=	processor(text=UpperCAmelCase     ,   images=UpperCAmelCase  )

														# For now the processor supports only ['input_ids', 'attention_mask', 'pixel_values']
														self.assertListEqual(list(inputs.keys()  )     ,   ["""input_ids""", """attention_mask""", """pixel_values"""]  )


