




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


UpperCAmelCase__						:       str							=    logging.get_logger(__name__)

UpperCAmelCase__						:       Dict							=    {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase__						:       List[Any]							=    {
    'vocab_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase__						:       Tuple							=    {
    'microsoft/layoutlm-base-uncased': 512,
    'microsoft/layoutlm-large-uncased': 512,
}

UpperCAmelCase__						:       Optional[Any]							=    {
    'microsoft/layoutlm-base-uncased': {'do_lower_case': True},
    'microsoft/layoutlm-large-uncased': {'do_lower_case': True},
}

class      UpperCAmelCase						(			A   ):
   __SCREAMING_SNAKE_CASE		:							List[Any]							=					VOCAB_FILES_NAMES
   __SCREAMING_SNAKE_CASE		:							Dict							=					PRETRAINED_VOCAB_FILES_MAP
   __SCREAMING_SNAKE_CASE		:							List[str]							=					PRETRAINED_INIT_CONFIGURATION
   __SCREAMING_SNAKE_CASE		:							Optional[int]							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __SCREAMING_SNAKE_CASE		:							List[Any]							=					LayoutLMTokenizer






   def __init__(	self   :       Dict						,					UpperCamelCase_   :       Optional[int]=None						,					UpperCamelCase_   :       List[str]=None						,					UpperCamelCase_   :       Optional[Any]=True						,					UpperCamelCase_   :       int="[UNK]"						,					UpperCamelCase_   :       Optional[Any]="[SEP]"						,					UpperCamelCase_   :       int="[PAD]"						,					UpperCamelCase_   :       Dict="[CLS]"						,					UpperCamelCase_   :       Optional[Any]="[MASK]"						,					UpperCamelCase_   :       Dict=True						,					UpperCamelCase_   :       int=None						,					**UpperCamelCase_   :       List[str]						,					)->			int:
     super().__init__(
         UpperCamelCase_						,					tokenizer_file=UpperCamelCase_						,					do_lower_case=UpperCamelCase_						,					unk_token=UpperCamelCase_						,					sep_token=UpperCamelCase_						,					pad_token=UpperCamelCase_						,					cls_token=UpperCamelCase_						,					mask_token=UpperCamelCase_						,					tokenize_chinese_chars=UpperCamelCase_						,					strip_accents=UpperCamelCase_						,					**UpperCamelCase_						,					)

     _lowercase: int						     =    json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
     if (
         normalizer_state.get('lowercase'						,					UpperCamelCase_     ) != do_lower_case
         or normalizer_state.get('strip_accents'						,					UpperCamelCase_     ) != strip_accents
         or normalizer_state.get('handle_chinese_chars'						,					UpperCamelCase_     ) != tokenize_chinese_chars
     ):
       _lowercase: Union[str, Any]						     =    getattr(UpperCamelCase_						,					normalizer_state.pop('type'     )     )
       _lowercase: List[str]						     =    do_lower_case
       _lowercase: Optional[int]						     =    strip_accents
       _lowercase: Union[str, Any]						     =    tokenize_chinese_chars
       _lowercase: Dict						     =    normalizer_class(**UpperCamelCase_     )

     _lowercase: List[str]						     =    do_lower_case






   def  _a						(	self   :       List[str]						,					UpperCamelCase_   :       List[str]						,					UpperCamelCase_   :       Optional[int]=None     )->			int:
     _lowercase: str						     =    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

     return output






   def  _a						(	self   :       Optional[Any]						,					UpperCamelCase_   :       List[int]						,					UpperCamelCase_   :       Optional[List[int]] = None     )->			List[int]:
     _lowercase: Optional[int]						     =    [self.sep_token_id]
     _lowercase: Optional[Any]						     =    [self.cls_token_id]
     if token_ids_a is None:
       return len(cls + token_ids_a + sep     ) * [0]
     return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]






   def  _a						(	self   :       int						,					UpperCamelCase_   :       str						,					UpperCamelCase_   :       Optional[str] = None     )->			Tuple[str]:
     _lowercase: Tuple						     =    self._tokenizer.model.save(UpperCamelCase_						,					name=UpperCamelCase_     )
     return tuple(UpperCamelCase_     )






import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin

class      UpperCAmelCase						(			A   ):
   __SCREAMING_SNAKE_CASE		:							Optional[Any]							=					"AutoFeatureExtractor"
   __SCREAMING_SNAKE_CASE		:							Optional[Any]							=					"Speech2Text2Tokenizer"






   def __init__(	self   :       Optional[Any]						,					UpperCamelCase_   :       Any						,					UpperCamelCase_   :       List[Any]     )->			Dict:
     super().__init__(UpperCamelCase_						,					UpperCamelCase_     )
     _lowercase: Optional[int]						     =    self.feature_extractor
     _lowercase: int						     =    False






   def __call__(	self   :       List[Any]						,					*UpperCamelCase_   :       Dict						,					**UpperCamelCase_   :       Optional[int]     )->			str:
     # For backward compatibility
     if self._in_target_context_manager:
       return self.current_processor(*UpperCamelCase_						,					**UpperCamelCase_     )

     if "raw_speech" in kwargs:
       warnings.warn('Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'     )
       _lowercase: Tuple						     =    kwargs.pop('raw_speech'     )
     else:
       _lowercase: Optional[Any]						     =    kwargs.pop('audio'						,					UpperCamelCase_     )
     _lowercase: int						     =    kwargs.pop('sampling_rate'						,					UpperCamelCase_     )
     _lowercase: Dict						     =    kwargs.pop('text'						,					UpperCamelCase_     )
     if len(UpperCamelCase_     ) > 0:
       _lowercase: List[Any]						     =    args[0]
       _lowercase: List[Any]						     =    args[1:]

     if audio is None and text is None:
       raise ValueError('You need to specify either an `audio` or `text` input to process.'     )

     if audio is not None:
       _lowercase: Optional[int]						     =    self.feature_extractor(UpperCamelCase_						,					*UpperCamelCase_						,					sampling_rate=UpperCamelCase_						,					**UpperCamelCase_     )
     if text is not None:
       _lowercase: str						     =    self.tokenizer(UpperCamelCase_						,					**UpperCamelCase_     )

     if text is None:
       return inputs
     elif audio is None:
       return encodings
     else:
       _lowercase: int						     =    encodings['input_ids']
       return inputs






   def  _a						(	self   :       Optional[int]						,					*UpperCamelCase_   :       Tuple						,					**UpperCamelCase_   :       Dict     )->			List[str]:
     return self.tokenizer.batch_decode(*UpperCamelCase_						,					**UpperCamelCase_     )






   def  _a						(	self   :       Tuple						,					*UpperCamelCase_   :       List[Any]						,					**UpperCamelCase_   :       Union[str, Any]     )->			Optional[int]:
     return self.tokenizer.decode(*UpperCamelCase_						,					**UpperCamelCase_     )






   @contextmanager
   def  _a						(	self   :       Tuple     )->			Union[str, Any]:
     warnings.warn(
         '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
         'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
         'your audio inputs, or in a separate call.'     )
     _lowercase: Any						     =    True
     _lowercase: Optional[Any]						     =    self.tokenizer
     yield
     _lowercase: Union[str, Any]						     =    self.feature_extractor
     _lowercase: str						     =    False
