



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


UpperCAmelCase :				Optional[int]							=logging.get_logger(__name__)

UpperCAmelCase :				List[str]							={"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase :				str							={
    """vocab_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase :				str							={
    """microsoft/layoutlm-base-uncased""": 512,
    """microsoft/layoutlm-large-uncased""": 512,
}

UpperCAmelCase :				Any							={
    """microsoft/layoutlm-base-uncased""": {"""do_lower_case""": True},
    """microsoft/layoutlm-large-uncased""": {"""do_lower_case""": True},
}


class    _lowercase (a_   ):
				'''simple docstring'''







				lowercase__     =    VOCAB_FILES_NAMES
				lowercase__     =    PRETRAINED_VOCAB_FILES_MAP
				lowercase__     =    PRETRAINED_INIT_CONFIGURATION
				lowercase__     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__     =    LayoutLMTokenizer

				def __init__(			self  ,			snake_case__=None  ,			snake_case__=None  ,			snake_case__=True  ,			snake_case__="[UNK]"  ,			snake_case__="[SEP]"  ,			snake_case__="[PAD]"  ,			snake_case__="[CLS]"  ,			snake_case__="[MASK]"  ,			snake_case__=True  ,			snake_case__=None  ,			**snake_case__  ,			):

									'''simple docstring'''


									super().__init__(
									    snake_case__  ,			tokenizer_file=snake_case__  ,			do_lower_case=snake_case__  ,			unk_token=snake_case__  ,			sep_token=snake_case__  ,			pad_token=snake_case__  ,			cls_token=snake_case__  ,			mask_token=snake_case__  ,			tokenize_chinese_chars=snake_case__  ,			strip_accents=snake_case__  ,			**snake_case__  ,			)

									UpperCamelCase_			= json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
									if (
									    normalizer_state.get("lowercase"  ,			snake_case__     ) != do_lower_case
									    or normalizer_state.get("strip_accents"  ,			snake_case__     ) != strip_accents
									    or normalizer_state.get("handle_chinese_chars"  ,			snake_case__     ) != tokenize_chinese_chars
									):
														UpperCamelCase_			= getattr(snake_case__  ,			normalizer_state.pop("type"     )     )
														UpperCamelCase_			= do_lower_case
														UpperCamelCase_			= strip_accents
														UpperCamelCase_			= tokenize_chinese_chars
														UpperCamelCase_			= normalizer_class(**snake_case__     )

									UpperCamelCase_			= do_lower_case

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__=None     ):

									'''simple docstring'''


									UpperCamelCase_			= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

									return output

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__ = None     ):

									'''simple docstring'''


									UpperCamelCase_			= [self.sep_token_id]
									UpperCamelCase_			= [self.cls_token_id]
									if token_ids_a is None:
														return len(cls + token_ids_a + sep     ) * [0]
									return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__ = None     ):

									'''simple docstring'''


									UpperCamelCase_			= self._tokenizer.model.save(snake_case__  ,			name=snake_case__     )
									return tuple(snake_case__     )






'''simple docstring'''


import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin






class 						lowerCAmelCase_	(       UpperCAmelCase_							):







  '''simple docstring'''

  UpperCamelCase_			:      Optional[int]										=				"""AutoFeatureExtractor"""
  UpperCamelCase_			:      List[Any]										=				"""Speech2Text2Tokenizer"""



  def __init__( self				:							str					,  SCREAMING_SNAKE_CASE_				:							str					,  SCREAMING_SNAKE_CASE_				:							int	)			->		Any:







       '''simple docstring'''



       super().__init__(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)
       A:		Union[str, Any]          =		self.feature_extractor
       A:		Dict          =		False



  def __call__( self				:							Any					,  *SCREAMING_SNAKE_CASE_				:							Any					,  **SCREAMING_SNAKE_CASE_				:							Optional[int]	)			->		Optional[Any]:







       '''simple docstring'''



       if self._in_target_context_manager:
            return self.current_processor(*SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_	)

       if "raw_speech" in kwargs:
            warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''	)
            A:		List[str]          =		kwargs.pop('''raw_speech'''	)
       else:
            A:		Any          =		kwargs.pop('''audio'''					,  SCREAMING_SNAKE_CASE_	)
       A:		List[Any]          =		kwargs.pop('''sampling_rate'''					,  SCREAMING_SNAKE_CASE_	)
       A:		Optional[int]          =		kwargs.pop('''text'''					,  SCREAMING_SNAKE_CASE_	)
       if len(SCREAMING_SNAKE_CASE_	) > 0:
            A:		str          =		args[0]
            A:		Tuple          =		args[1:]

       if audio is None and text is None:
            raise ValueError('''You need to specify either an `audio` or `text` input to process.'''	)

       if audio is not None:
            A:		Tuple          =		self.feature_extractor(SCREAMING_SNAKE_CASE_					,  *SCREAMING_SNAKE_CASE_					,  sampling_rate=SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_	)
       if text is not None:
            A:		List[Any]          =		self.tokenizer(SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_	)

       if text is None:
            return inputs
       elif audio is None:
            return encodings
       else:
            A:		str          =		encodings['''input_ids''']
            return inputs



  def 					_snake_case     ( self				:							Tuple					,  *SCREAMING_SNAKE_CASE_				:							int					,  **SCREAMING_SNAKE_CASE_				:							Union[str, Any]	)			->		str:







       '''simple docstring'''



       return self.tokenizer.batch_decode(*SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_	)



  def 					_snake_case     ( self				:							Optional[int]					,  *SCREAMING_SNAKE_CASE_				:							str					,  **SCREAMING_SNAKE_CASE_				:							int	)			->		Union[str, Any]:







       '''simple docstring'''



       return self.tokenizer.decode(*SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_	)




  @contextmanager
  def 					_snake_case     ( self				:							str	)			->		int:







       '''simple docstring'''



       warnings.warn(
           '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
           '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
           '''your audio inputs, or in a separate call.'''	)
       A:		List[Any]          =		True
       A:		List[str]          =		self.tokenizer
       yield
       A:		List[Any]          =		self.feature_extractor
       A:		int          =		False






