



import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType







class       _snake_case				(     __snake_case ):






   '''simple docstring'''






   A__      : str       =						["image_processor", "tokenizer"]
   A__      : str       =						"LayoutLMv2ImageProcessor"
   A__      : str       =						("LayoutLMv2Tokenizer", "LayoutLMv2TokenizerFast")


   def __init__(    self:     str  ,lowerCamelCase_:     Optional[int]=None  ,lowerCamelCase_:     str=None  ,**lowerCamelCase_:     Dict							)	->      Optional[Any]:
       UpperCAmelCase_     :       str  		=					None
       if "feature_extractor" in kwargs:
           warnings.warn(
               """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
               """ instead."""  ,lowerCamelCase_  ,)
           UpperCAmelCase_     :       Union[str, Any]  		=					kwargs.pop("""feature_extractor"""							)

       UpperCAmelCase_     :       Dict  		=					image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
           raise ValueError("""You need to specify an `image_processor`."""							)
       if tokenizer is None:
           raise ValueError("""You need to specify a `tokenizer`."""							)

       super().__init__(lowerCamelCase_  ,lowerCamelCase_							)


   def __call__(    self:     Tuple  ,lowerCamelCase_:     Optional[int]  ,lowerCamelCase_:     Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None  ,lowerCamelCase_:     Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None  ,lowerCamelCase_:     Union[List[List[int]], List[List[List[int]]]] = None  ,lowerCamelCase_:     Optional[Union[List[int], List[List[int]]]] = None  ,lowerCamelCase_:     bool = True  ,lowerCamelCase_:     Union[bool, str, PaddingStrategy] = False  ,lowerCamelCase_:     Union[bool, str, TruncationStrategy] = False  ,lowerCamelCase_:     Optional[int] = None  ,lowerCamelCase_:     int = 0  ,lowerCamelCase_:     Optional[int] = None  ,lowerCamelCase_:     Optional[bool] = None  ,lowerCamelCase_:     Optional[bool] = None  ,lowerCamelCase_:     bool = False  ,lowerCamelCase_:     bool = False  ,lowerCamelCase_:     bool = False  ,lowerCamelCase_:     bool = False  ,lowerCamelCase_:     bool = True  ,lowerCamelCase_:     Optional[Union[str, TensorType]] = None  ,**lowerCamelCase_:     Optional[int]  ,)	->      BatchEncoding:
       # verify input
       if self.image_processor.apply_ocr and (boxes is not None):
           raise ValueError(
               """You cannot provide bounding boxes if you initialized the image processor with apply_ocr set to True."""							)

       if self.image_processor.apply_ocr and (word_labels is not None):
           raise ValueError(
               """You cannot provide word labels if you initialized the image processor with apply_ocr set to True."""							)

       if return_overflowing_tokens is True and return_offsets_mapping is False:
           raise ValueError("""You cannot return overflowing tokens without returning the offsets mapping."""							)

       # first, apply the image processor
       UpperCAmelCase_     :       Optional[int]  		=					self.image_processor(images=lowerCamelCase_  ,return_tensors=lowerCamelCase_							)

       # second, apply the tokenizer
       if text is not None and self.image_processor.apply_ocr and text_pair is None:
           if isinstance(lowerCamelCase_  ,lowerCamelCase_							):
               UpperCAmelCase_     :       int  		=					[text]  # add batch dimension (as the image processor always adds a batch dimension)
           UpperCAmelCase_     :       str  		=					features["""words"""]

       UpperCAmelCase_     :       str  		=					self.tokenizer(
           text=text if text is not None else features["""words"""]  ,text_pair=text_pair if text_pair is not None else None  ,boxes=boxes if boxes is not None else features["""boxes"""]  ,word_labels=lowerCamelCase_  ,add_special_tokens=lowerCamelCase_  ,padding=lowerCamelCase_  ,truncation=lowerCamelCase_  ,max_length=lowerCamelCase_  ,stride=lowerCamelCase_  ,pad_to_multiple_of=lowerCamelCase_  ,return_token_type_ids=lowerCamelCase_  ,return_attention_mask=lowerCamelCase_  ,return_overflowing_tokens=lowerCamelCase_  ,return_special_tokens_mask=lowerCamelCase_  ,return_offsets_mapping=lowerCamelCase_  ,return_length=lowerCamelCase_  ,verbose=lowerCamelCase_  ,return_tensors=lowerCamelCase_  ,**lowerCamelCase_  ,)

       # add pixel values
       UpperCAmelCase_     :       Optional[int]  		=					features.pop("""pixel_values"""							)
       if return_overflowing_tokens is True:
           UpperCAmelCase_     :       str  		=					self.get_overflowing_images(lowerCamelCase_  ,encoded_inputs["""overflow_to_sample_mapping"""]							)
       UpperCAmelCase_     :       Tuple  		=					images

       return encoded_inputs


   def       A__    (    self:     Dict  ,lowerCamelCase_:     str  ,lowerCamelCase_:     Optional[int]							)	->      str:
       # in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
       UpperCAmelCase_     :       List[Any]  		=					[]
       for sample_idx in overflow_to_sample_mapping:
           images_with_overflow.append(images[sample_idx]							)

       if len(lowerCamelCase_							) != len(lowerCamelCase_							):
           raise ValueError(
               """Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"""
               F''' {len(lowerCamelCase_							)} and {len(lowerCamelCase_							)}'''							)

       return images_with_overflow


   def       A__    (    self:     Optional[int]  ,*lowerCamelCase_:     Union[str, Any]  ,**lowerCamelCase_:     List[str]							)	->      Optional[Any]:
       return self.tokenizer.batch_decode(*lowerCamelCase_  ,**lowerCamelCase_							)


   def       A__    (    self:     Tuple  ,*lowerCamelCase_:     Dict  ,**lowerCamelCase_:     int							)	->      str:
       return self.tokenizer.decode(*lowerCamelCase_  ,**lowerCamelCase_							)


   @property
   def       A__    (    self:     str							)	->      Optional[Any]:
       return ["input_ids", "bbox", "token_type_ids", "attention_mask", "image"]


   @property
   def       A__    (    self:     Any							)	->      Tuple:
       warnings.warn(
           """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""  ,lowerCamelCase_  ,)
       return self.image_processor_class






   @property
   def       A__    (    self:     Union[str, Any]							)	->      List[Any]:
       warnings.warn(
           """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""  ,lowerCamelCase_  ,)
       return self.image_processor






'''simple docstring'''
import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A				: List[str]					      =			logging.get_logger(__name__)


A				: Optional[int]					      =			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
A				: Tuple					      =			{
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
}

A				: Optional[int]					      =			{
    """RUCAIBox/mvp""": 1_024,
}
@lru_cache()
def 							snake_case_    (  ):



    """simple docstring"""




    __lowercase     							=		(
        list(range(ord("""!"""     )       ,ord("""~"""     ) + 1     )     ) + list(range(ord("""¡"""     )       ,ord("""¬"""     ) + 1     )     ) + list(range(ord("""®"""     )       ,ord("""ÿ"""     ) + 1     )     )
    )
    __lowercase     							=		bs[:]
    __lowercase     							=		0
    for b in range(2**8     ):
        if b not in bs:
            bs.append(a__     )
            cs.append(2**8 + n     )
            n += 1
    __lowercase     							=		[chr(a__     ) for n in cs]
    return dict(zip(a__       ,a__     )     )
def 							snake_case_    (  a__						:      Dict     ):



    """simple docstring"""




    __lowercase     							=		set()
    __lowercase     							=		word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)     )
        __lowercase     							=		char
    return pairs






class 			SCREAMING_SNAKE_CASE(							__A	):
       snake_case_					:	int       =   VOCAB_FILES_NAMES
       snake_case_					:	List[Any]       =   PRETRAINED_VOCAB_FILES_MAP
       snake_case_					:	List[Any]       =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       snake_case_					:	Dict       =   ["""input_ids""", """attention_mask"""]

       def __init__(       self   ,						lowerCamelCase__   ,						lowerCamelCase__   ,						lowerCamelCase__="replace"   ,						lowerCamelCase__="<s>"   ,						lowerCamelCase__="</s>"   ,						lowerCamelCase__="</s>"   ,						lowerCamelCase__="<s>"   ,						lowerCamelCase__="<unk>"   ,						lowerCamelCase__="<pad>"   ,						lowerCamelCase__="<mask>"   ,						lowerCamelCase__=False   ,						**lowerCamelCase__   ,						)       ->   Union[str, Any]:

           """simple docstring"""



           __lowercase     							=		AddedToken(lowerCamelCase__   ,						lstrip=lowerCamelCase__   ,						rstrip=lowerCamelCase__	) if isinstance(lowerCamelCase__   ,						lowerCamelCase__	) else bos_token
           __lowercase     							=		AddedToken(lowerCamelCase__   ,						lstrip=lowerCamelCase__   ,						rstrip=lowerCamelCase__	) if isinstance(lowerCamelCase__   ,						lowerCamelCase__	) else eos_token
           __lowercase     							=		AddedToken(lowerCamelCase__   ,						lstrip=lowerCamelCase__   ,						rstrip=lowerCamelCase__	) if isinstance(lowerCamelCase__   ,						lowerCamelCase__	) else sep_token
           __lowercase     							=		AddedToken(lowerCamelCase__   ,						lstrip=lowerCamelCase__   ,						rstrip=lowerCamelCase__	) if isinstance(lowerCamelCase__   ,						lowerCamelCase__	) else cls_token
           __lowercase     							=		AddedToken(lowerCamelCase__   ,						lstrip=lowerCamelCase__   ,						rstrip=lowerCamelCase__	) if isinstance(lowerCamelCase__   ,						lowerCamelCase__	) else unk_token
           __lowercase     							=		AddedToken(lowerCamelCase__   ,						lstrip=lowerCamelCase__   ,						rstrip=lowerCamelCase__	) if isinstance(lowerCamelCase__   ,						lowerCamelCase__	) else pad_token

           # Mask token behave like a normal word, i.e. include the space before it
           __lowercase     							=		AddedToken(lowerCamelCase__   ,						lstrip=lowerCamelCase__   ,						rstrip=lowerCamelCase__	) if isinstance(lowerCamelCase__   ,						lowerCamelCase__	) else mask_token

           super().__init__(
               errors=lowerCamelCase__   ,						bos_token=lowerCamelCase__   ,						eos_token=lowerCamelCase__   ,						unk_token=lowerCamelCase__   ,						sep_token=lowerCamelCase__   ,						cls_token=lowerCamelCase__   ,						pad_token=lowerCamelCase__   ,						mask_token=lowerCamelCase__   ,						add_prefix_space=lowerCamelCase__   ,						**lowerCamelCase__   ,						)
           with open(lowerCamelCase__   ,						encoding="""utf-8"""	) as vocab_handle:
               __lowercase     							=		json.load(lowerCamelCase__	)
           __lowercase     							=		{v: k for k, v in self.encoder.items()}
           __lowercase     							=		errors  # how to handle errors in decoding
           __lowercase     							=		bytes_to_unicode()
           __lowercase     							=		{v: k for k, v in self.byte_encoder.items()}
           with open(lowerCamelCase__   ,						encoding="""utf-8"""	) as merges_handle:
               __lowercase     							=		merges_handle.read().split("""\n"""	)[1:-1]
           __lowercase     							=		[tuple(merge.split()	) for merge in bpe_merges]
           __lowercase     							=		dict(zip(lowerCamelCase__   ,						range(len(lowerCamelCase__	)	)	)	)
           __lowercase     							=		{}
           __lowercase     							=		add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           __lowercase     							=		re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""	)

       @property
       def        snake_case__     (       self	)       ->   List[str]:

           """simple docstring"""



           return len(self.encoder	)

       def        snake_case__     (       self	)       ->   Optional[int]:

           """simple docstring"""



           return dict(self.encoder   ,						**self.added_tokens_encoder	)

       def        snake_case__     (       self   ,						lowerCamelCase__	)       ->   Any:

           """simple docstring"""



           if token in self.cache:
               return self.cache[token]
           __lowercase     							=		tuple(lowerCamelCase__	)
           __lowercase     							=		get_pairs(lowerCamelCase__	)

           if not pairs:
               return token

           while True:
               __lowercase     							=		min(lowerCamelCase__   ,						key=lambda lowerCamelCase__	: self.bpe_ranks.get(lowerCamelCase__   ,						float("""inf"""	)	)	)
               if bigram not in self.bpe_ranks:
                   break
               __lowercase							,__lowercase     							=		bigram
               __lowercase     							=		[]
               __lowercase     							=		0
               while i < len(lowerCamelCase__	):
                   try:
                       __lowercase     							=		word.index(lowerCamelCase__   ,						lowerCamelCase__	)
                   except ValueError:
                       new_word.extend(word[i:]	)
                       break
                   else:
                       new_word.extend(word[i:j]	)
                       __lowercase     							=		j

                   if word[i] == first and i < len(lowerCamelCase__	) - 1 and word[i + 1] == second:
                       new_word.append(first + second	)
                       i += 2
                   else:
                       new_word.append(word[i]	)
                       i += 1
               __lowercase     							=		tuple(lowerCamelCase__	)
               __lowercase     							=		new_word
               if len(lowerCamelCase__	) == 1:
                   break
               else:
                   __lowercase     							=		get_pairs(lowerCamelCase__	)
           __lowercase     							=		""" """.join(lowerCamelCase__	)
           __lowercase     							=		word
           return word

       def        snake_case__     (       self   ,						lowerCamelCase__	)       ->   Union[str, Any]:

           """simple docstring"""



           __lowercase     							=		[]
           for token in re.findall(self.pat   ,						lowerCamelCase__	):
               __lowercase     							=		"""""".join(
                   self.byte_encoder[b] for b in token.encode("""utf-8"""	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase__	).split(""" """	)	)
           return bpe_tokens

       def        snake_case__     (       self   ,						lowerCamelCase__	)       ->   int:

           """simple docstring"""



           return self.encoder.get(lowerCamelCase__   ,						self.encoder.get(self.unk_token	)	)

       def        snake_case__     (       self   ,						lowerCamelCase__	)       ->   Dict:

           """simple docstring"""



           return self.decoder.get(lowerCamelCase__	)

       def        snake_case__     (       self   ,						lowerCamelCase__	)       ->   Tuple:

           """simple docstring"""



           __lowercase     							=		"""""".join(lowerCamelCase__	)
           __lowercase     							=		bytearray([self.byte_decoder[c] for c in text]	).decode("""utf-8"""   ,						errors=self.errors	)
           return text

       def        snake_case__     (       self   ,						lowerCamelCase__   ,						lowerCamelCase__ = None	)       ->   Tuple[str]:

           """simple docstring"""



           if not os.path.isdir(lowerCamelCase__	):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory'	)
               return
           __lowercase     							=		os.path.join(
               lowerCamelCase__   ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)
           __lowercase     							=		os.path.join(
               lowerCamelCase__   ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]	)

           with open(lowerCamelCase__   ,						"""w"""   ,						encoding="""utf-8"""	) as f:
               f.write(json.dumps(self.encoder   ,						indent=2   ,						sort_keys=lowerCamelCase__   ,						ensure_ascii=lowerCamelCase__	) + """\n"""	)

           __lowercase     							=		0
           with open(lowerCamelCase__   ,						"""w"""   ,						encoding="""utf-8"""	) as writer:
               writer.write("""#version: 0.2\n"""	)
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,						key=lambda lowerCamelCase__	: kv[1]	):
                   if index != token_index:
                       logger.warning(
                           F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                           """ Please check that the tokenizer is not corrupted!"""	)
                       __lowercase     							=		token_index
                   writer.write(""" """.join(lowerCamelCase__	) + """\n"""	)
                   index += 1

           return vocab_file, merge_file

       def        snake_case__     (       self   ,						lowerCamelCase__   ,						lowerCamelCase__ = None	)       ->   List[int]:

           """simple docstring"""



           if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           __lowercase     							=		[self.cls_token_id]
           __lowercase     							=		[self.sep_token_id]
           return cls + token_ids_a + sep + sep + token_ids_a + sep

       def        snake_case__     (       self   ,						lowerCamelCase__   ,						lowerCamelCase__ = None   ,						lowerCamelCase__ = False	)       ->   List[int]:

           """simple docstring"""



           if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowerCamelCase__   ,						token_ids_a=lowerCamelCase__   ,						already_has_special_tokens=lowerCamelCase__	)

           if token_ids_a is None:
               return [1] + ([0] * len(lowerCamelCase__	)) + [1]
           return [1] + ([0] * len(lowerCamelCase__	)) + [1, 1] + ([0] * len(lowerCamelCase__	)) + [1]

       def        snake_case__     (       self   ,						lowerCamelCase__   ,						lowerCamelCase__ = None	)       ->   List[int]:

           """simple docstring"""



           __lowercase     							=		[self.sep_token_id]
           __lowercase     							=		[self.cls_token_id]

           if token_ids_a is None:
               return len(cls + token_ids_a + sep	) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]




       def        snake_case__     (       self   ,						lowerCamelCase__   ,						lowerCamelCase__=False   ,						**lowerCamelCase__	)       ->   List[str]:

           """simple docstring"""



           __lowercase     							=		kwargs.pop("""add_prefix_space"""   ,						self.add_prefix_space	)
           if (is_split_into_words or add_prefix_space) and (len(lowerCamelCase__	) > 0 and not text[0].isspace()):
               __lowercase     							=		""" """ + text
           return (text, kwargs)




