



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


_lowercase				:					int     					=logging.get_logger(__name__)

_lowercase				:					Optional[Any]     					={"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase				:					Union[str, Any]     					={
    """vocab_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

_lowercase				:					List[str]     					={
    """microsoft/layoutlm-base-uncased""": 5_12,
    """microsoft/layoutlm-large-uncased""": 5_12,
}

_lowercase				:					str     					={
    """microsoft/layoutlm-base-uncased""": {"""do_lower_case""": True},
    """microsoft/layoutlm-large-uncased""": {"""do_lower_case""": True},
}
class lowerCAmelCase_       (   __SCREAMING_SNAKE_CASE			):







	'''simple docstring'''




	A_       :     Optional[Any]									=		VOCAB_FILES_NAMES
	A_       :     str									=		PRETRAINED_VOCAB_FILES_MAP
	A_       :     str									=		PRETRAINED_INIT_CONFIGURATION
	A_       :     Any									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A_       :     Optional[int]									=		LayoutLMTokenizer

	def __init__(	self   ,       lowerCamelCase=None   ,       lowerCamelCase=None   ,       lowerCamelCase=True   ,       lowerCamelCase="[UNK]"   ,       lowerCamelCase="[SEP]"   ,       lowerCamelCase="[PAD]"   ,       lowerCamelCase="[CLS]"   ,       lowerCamelCase="[MASK]"   ,       lowerCamelCase=True   ,       lowerCamelCase=None   ,       **lowerCamelCase   ,       ):



				'''simple docstring'''


				super().__init__(
				    _A   ,       tokenizer_file=_A   ,       do_lower_case=_A   ,       unk_token=_A   ,       sep_token=_A   ,       pad_token=_A   ,       cls_token=_A   ,       mask_token=_A   ,       tokenize_chinese_chars=_A   ,       strip_accents=_A   ,       **_A   ,       )

				a__					=  json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
				if (
				    normalizer_state.get("""lowercase"""   ,       _A       ) != do_lower_case
				    or normalizer_state.get("""strip_accents"""   ,       _A       ) != strip_accents
				    or normalizer_state.get("""handle_chinese_chars"""   ,       _A       ) != tokenize_chinese_chars
				):
							a__					=  getattr(_A   ,       normalizer_state.pop("""type"""       )       )
							a__					=  do_lower_case
							a__					=  strip_accents
							a__					=  tokenize_chinese_chars
							a__					=  normalizer_class(**_A       )

				a__					=  do_lower_case

	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase=None       ):



				'''simple docstring'''


				a__					=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output

	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase = None       ):



				'''simple docstring'''


				a__					=  [self.sep_token_id]
				a__					=  [self.cls_token_id]
				if token_ids_a is None:
							return len(cls + token_ids_a + sep       ) * [0]
				return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase = None       ):



				'''simple docstring'''


				a__					=  self._tokenizer.model.save(_A   ,       name=_A       )
				return tuple(_A       )



import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto.configuration_auto import AutoConfig


__UpperCAmelCase   						=     logging.get_logger(__name__)





class 				UpperCamelCase__		(	__SCREAMING_SNAKE_CASE	):






       """simple docstring"""


       UpperCAmelCase_				       ="speech-encoder-decoder"
       UpperCAmelCase_				       =True



       def __init__(							self ,     **_A	)				->				Dict:
          super().__init__(**_A	)
          if "encoder" not in kwargs or "decoder" not in kwargs:
             raise ValueError(
                 F'''A configuraton of type {self.model_type} cannot be instantiated because not both `encoder` and'''
                 F''' `decoder` sub-configurations are passed, but only {kwargs}'''	)

          SCREAMING_SNAKE_CASE_	=      kwargs.pop('''encoder'''	)
          SCREAMING_SNAKE_CASE_	=      encoder_config.pop('''model_type'''	)
          SCREAMING_SNAKE_CASE_	=      kwargs.pop('''decoder'''	)
          SCREAMING_SNAKE_CASE_	=      decoder_config.pop('''model_type'''	)

          SCREAMING_SNAKE_CASE_	=      AutoConfig.for_model(_A ,     **_A	)
          SCREAMING_SNAKE_CASE_	=      AutoConfig.for_model(_A ,     **_A	)
          SCREAMING_SNAKE_CASE_	=      True



       @classmethod
       def     _UpperCamelCase	(							cls ,     _A ,     _A ,     **_A	)				->				PretrainedConfig:
          logger.info('''Setting `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config'''	)
          SCREAMING_SNAKE_CASE_	=      True
          SCREAMING_SNAKE_CASE_	=      True

          return cls(encoder=encoder_config.to_dict() ,     decoder=decoder_config.to_dict() ,     **_A	)




       def     _UpperCamelCase	(							self	)				->				Union[str, Any]:
          SCREAMING_SNAKE_CASE_	=      copy.deepcopy(self.__dict__	)
          SCREAMING_SNAKE_CASE_	=      self.encoder.to_dict()
          SCREAMING_SNAKE_CASE_	=      self.decoder.to_dict()
          SCREAMING_SNAKE_CASE_	=      self.__class__.model_type
          return output


