
"""simple docstring"""




import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available


if is_torch_available():
					from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
					__lowerCAmelCase							:  Optional[Any] 				=		False

if is_vision_available():
					from PIL import Image

					from transformers import (
					    AutoProcessor,
					    PixaStructImageProcessor,
					    PixaStructProcessor,
					    PreTrainedTokenizerFast,
					    TaTokenizer,
					)
@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11					,   reason='`Pix2StructImageProcessor` requires `torch>=1.11.0`.'					,   )
@require_vision
@require_torch
class SCREAMING_SNAKE_CASE       (      unittest.TestCase					):






	'''simple docstring'''







	def    _UpperCamelCase							(  self :List[str]			)			->   Dict:

			'''simple docstring'''






			a__   		=     tempfile.mkdtemp()

			a__   		=     PixaStructImageProcessor()
			a__   		=     TaTokenizer.from_pretrained('''t5-small'''			)

			a__   		=     PixaStructProcessor(__magic_name__       ,    __magic_name__			)

			processor.save_pretrained(self.tmpdirname			)


	def    _UpperCamelCase							(  self :List[Any]       ,    **__magic_name__ :int			)			->   Union[str, Any]:

			'''simple docstring'''






			return AutoProcessor.from_pretrained(self.tmpdirname       ,    **__magic_name__			).tokenizer


	def    _UpperCamelCase							(  self :List[str]       ,    **__magic_name__ :int			)			->   Optional[Any]:

			'''simple docstring'''






			return AutoProcessor.from_pretrained(self.tmpdirname       ,    **__magic_name__			).image_processor


	def    _UpperCamelCase							(  self :Optional[Any]			)			->   Any:

			'''simple docstring'''






			shutil.rmtree(self.tmpdirname			)


	def    _UpperCamelCase							(  self :Tuple			)			->   Union[str, Any]:

			'''simple docstring'''






			a__   		=     [np.random.randint(255       ,    size=(3, 30, 400)       ,    dtype=np.uinta			)]

			a__   		=     [Image.fromarray(np.moveaxis(__magic_name__       ,    0       ,    -1			)			) for x in image_inputs]

			return image_inputs


	def    _UpperCamelCase							(  self :List[str]			)			->   Optional[Any]:

			'''simple docstring'''






			a__   		=     PixaStructProcessor(tokenizer=self.get_tokenizer()       ,    image_processor=self.get_image_processor()			)
			processor.save_pretrained(self.tmpdirname			)

			a__   		=     self.get_tokenizer(bos_token='''(BOS)'''       ,    eos_token='''(EOS)'''			)
			a__   		=     self.get_image_processor(do_normalize=__magic_name__       ,    padding_value=1.0			)

			a__   		=     PixaStructProcessor.from_pretrained(
			    self.tmpdirname       ,    bos_token='''(BOS)'''       ,    eos_token='''(EOS)'''       ,    do_normalize=__magic_name__       ,    padding_value=1.0			)

			self.assertEqual(processor.tokenizer.get_vocab()       ,    tokenizer_add_kwargs.get_vocab()			)
			self.assertIsInstance(processor.tokenizer       ,    __magic_name__			)

			self.assertEqual(processor.image_processor.to_json_string()       ,    image_processor_add_kwargs.to_json_string()			)
			self.assertIsInstance(processor.image_processor       ,    __magic_name__			)


	def    _UpperCamelCase							(  self :Dict			)			->   Dict:

			'''simple docstring'''






			a__   		=     self.get_image_processor()
			a__   		=     self.get_tokenizer()

			a__   		=     PixaStructProcessor(tokenizer=__magic_name__       ,    image_processor=__magic_name__			)

			a__   		=     self.prepare_image_inputs()

			a__   		=     image_processor(__magic_name__       ,    return_tensors='''np'''			)
			a__   		=     processor(images=__magic_name__       ,    return_tensors='''np'''			)

			for key in input_feat_extract.keys():
					self.assertAlmostEqual(input_feat_extract[key].sum()       ,    input_processor[key].sum()       ,    delta=1e-2			)


	def    _UpperCamelCase							(  self :Union[str, Any]			)			->   Dict:

			'''simple docstring'''






			a__   		=     self.get_image_processor()
			a__   		=     self.get_tokenizer()

			a__   		=     PixaStructProcessor(tokenizer=__magic_name__       ,    image_processor=__magic_name__			)

			a__   		=     '''lower newer'''

			a__   		=     processor(text=__magic_name__			)

			a__   		=     tokenizer(__magic_name__       ,    return_token_type_ids=__magic_name__       ,    add_special_tokens=__magic_name__			)

			for key in encoded_tok.keys():
					self.assertListEqual(encoded_tok[key]       ,    encoded_processor[key]			)


	def    _UpperCamelCase							(  self :Optional[int]			)			->   List[Any]:

			'''simple docstring'''






			a__   		=     self.get_image_processor()
			a__   		=     self.get_tokenizer()

			a__   		=     PixaStructProcessor(tokenizer=__magic_name__       ,    image_processor=__magic_name__			)

			a__   		=     '''lower newer'''
			a__   		=     self.prepare_image_inputs()

			a__   		=     processor(text=__magic_name__       ,    images=__magic_name__			)

			self.assertListEqual(
			    list(inputs.keys()			)       ,    ['''flattened_patches''', '''attention_mask''', '''decoder_attention_mask''', '''decoder_input_ids''']			)

			# test if it raises when no input is passed
			with pytest.raises(__magic_name__			):
					processor()


	def    _UpperCamelCase							(  self :Any			)			->   List[str]:

			'''simple docstring'''






			a__   		=     self.get_image_processor()
			a__   		=     self.get_tokenizer()

			a__   		=     PixaStructProcessor(tokenizer=__magic_name__       ,    image_processor=__magic_name__			)

			a__   		=     '''lower newer'''
			a__   		=     self.prepare_image_inputs()

			a__   		=     processor(text=__magic_name__       ,    images=__magic_name__			)

			a__   		=     [512, 1024, 2048, 4096]
			a__   		=     [770, 770, 770, 770]
			# with text
			for i, max_patch in enumerate(__magic_name__			):
					a__   		=     processor(text=__magic_name__       ,    images=__magic_name__       ,    max_patches=__magic_name__			)
					self.assertEqual(inputs['''flattened_patches'''][0].shape[0]       ,    __magic_name__			)
					self.assertEqual(inputs['''flattened_patches'''][0].shape[1]       ,    expected_hidden_size[i]			)

			# without text input
			for i, max_patch in enumerate(__magic_name__			):
					a__   		=     processor(images=__magic_name__       ,    max_patches=__magic_name__			)
					self.assertEqual(inputs['''flattened_patches'''][0].shape[0]       ,    __magic_name__			)
					self.assertEqual(inputs['''flattened_patches'''][0].shape[1]       ,    expected_hidden_size[i]			)


	def    _UpperCamelCase							(  self :Union[str, Any]			)			->   List[Any]:

			'''simple docstring'''






			a__   		=     self.get_image_processor()
			a__   		=     self.get_tokenizer()

			a__   		=     PixaStructProcessor(tokenizer=__magic_name__       ,    image_processor=__magic_name__			)

			a__   		=     [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

			a__   		=     processor.batch_decode(__magic_name__			)
			a__   		=     tokenizer.batch_decode(__magic_name__			)

			self.assertListEqual(__magic_name__       ,    __magic_name__			)






	def    _UpperCamelCase							(  self :Optional[Any]			)			->   str:

			'''simple docstring'''






			a__   		=     self.get_image_processor()
			a__   		=     self.get_tokenizer()

			a__   		=     PixaStructProcessor(tokenizer=__magic_name__       ,    image_processor=__magic_name__			)

			a__   		=     '''lower newer'''
			a__   		=     self.prepare_image_inputs()

			a__   		=     processor(text=__magic_name__       ,    images=__magic_name__			)

			# For now the processor supports only ["flattened_patches", "input_ids", "attention_mask", "decoder_attention_mask"]
			self.assertListEqual(
			    list(inputs.keys()			)       ,    ['''flattened_patches''', '''attention_mask''', '''decoder_attention_mask''', '''decoder_input_ids''']			)

			a__   		=     processor(text=__magic_name__			)

			# For now the processor supports only ["flattened_patches", "input_ids", "attention_mask", "decoder_attention_mask"]
			self.assertListEqual(list(inputs.keys()			)       ,    ['''input_ids''', '''attention_mask''']			)






"""simple docstring"""




from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy
class SCREAMING_SNAKE_CASE       (      lowerCAmelCase					):






	'''simple docstring'''







	snake_case__							:		Optional[Any]         =		'MarkupLMFeatureExtractor'
	snake_case__							:		Tuple         =		('MarkupLMTokenizer', 'MarkupLMTokenizerFast')
	snake_case__							:		Dict         =		True


	def __call__(  self :Any       ,    __magic_name__ :Dict=None       ,    __magic_name__ :Dict=None       ,    __magic_name__ :List[str]=None       ,    __magic_name__ :List[str]=None       ,    __magic_name__ :Tuple=None       ,    __magic_name__ :bool = True       ,    __magic_name__ :Union[bool, str, PaddingStrategy] = False       ,    __magic_name__ :Union[bool, str, TruncationStrategy] = None       ,    __magic_name__ :Optional[int] = None       ,    __magic_name__ :int = 0       ,    __magic_name__ :Optional[int] = None       ,    __magic_name__ :Optional[bool] = None       ,    __magic_name__ :Optional[bool] = None       ,    __magic_name__ :bool = False       ,    __magic_name__ :bool = False       ,    __magic_name__ :bool = False       ,    __magic_name__ :bool = False       ,    __magic_name__ :bool = True       ,    __magic_name__ :Optional[Union[str, TensorType]] = None       ,    **__magic_name__ :Optional[Any]       ,    )			->   BatchEncoding:

			'''simple docstring'''






			if self.parse_html:
					if html_strings is None:
							raise ValueError('''Make sure to pass HTML strings in case `parse_html` is set to `True`'''			)

					if nodes is not None or xpaths is not None or node_labels is not None:
							raise ValueError(
							    '''Please don\'t pass nodes, xpaths nor node labels in case `parse_html` is set to `True`'''			)

					a__   		=     self.feature_extractor(__magic_name__			)
					a__   		=     features['''nodes''']
					a__   		=     features['''xpaths''']
			else:
					if html_strings is not None:
							raise ValueError('''You have passed HTML strings but `parse_html` is set to `False`.'''			)
					if nodes is None or xpaths is None:
							raise ValueError('''Make sure to pass nodes and xpaths in case `parse_html` is set to `False`'''			)

        # # second, apply the tokenizer
			if questions is not None and self.parse_html:
					if isinstance(__magic_name__       ,    __magic_name__			):
							a__   		=     [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

			a__   		=     self.tokenizer(
			    text=questions if questions is not None else nodes       ,    text_pair=nodes if questions is not None else None       ,    xpaths=__magic_name__       ,    node_labels=__magic_name__       ,    add_special_tokens=__magic_name__       ,    padding=__magic_name__       ,    truncation=__magic_name__       ,    max_length=__magic_name__       ,    stride=__magic_name__       ,    pad_to_multiple_of=__magic_name__       ,    return_token_type_ids=__magic_name__       ,    return_attention_mask=__magic_name__       ,    return_overflowing_tokens=__magic_name__       ,    return_special_tokens_mask=__magic_name__       ,    return_offsets_mapping=__magic_name__       ,    return_length=__magic_name__       ,    verbose=__magic_name__       ,    return_tensors=__magic_name__       ,    **__magic_name__       ,    )

			return encoded_inputs


	def    _UpperCamelCase							(  self :Tuple       ,    *__magic_name__ :Optional[int]       ,    **__magic_name__ :Any			)			->   int:

			'''simple docstring'''






			return self.tokenizer.batch_decode(*__magic_name__       ,    **__magic_name__			)


	def    _UpperCamelCase							(  self :Tuple       ,    *__magic_name__ :Union[str, Any]       ,    **__magic_name__ :Union[str, Any]			)			->   Dict:

			'''simple docstring'''






			return self.tokenizer.decode(*__magic_name__       ,    **__magic_name__			)






	@property
	def    _UpperCamelCase							(  self :Optional[int]			)			->   Union[str, Any]:

			'''simple docstring'''






			a__   		=     self.tokenizer.model_input_names
			return tokenizer_input_names




