




"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


_SCREAMING_SNAKE_CASE						 =     logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE						 =     {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_SCREAMING_SNAKE_CASE						 =     {
    """vocab_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """microsoft/layoutlm-base-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"""
        ),
        """microsoft/layoutlm-large-uncased""": (
            """https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

_SCREAMING_SNAKE_CASE						 =     {
    """microsoft/layoutlm-base-uncased""": 5_1_2,
    """microsoft/layoutlm-large-uncased""": 5_1_2,
}

_SCREAMING_SNAKE_CASE						 =     {
    """microsoft/layoutlm-base-uncased""": {"""do_lower_case""": True},
    """microsoft/layoutlm-large-uncased""": {"""do_lower_case""": True},
}







class  _SCREAMING_SNAKE_CASE       (     SCREAMING_SNAKE_CASE__				):



 '''simple docstring'''





 __UpperCAmelCase		   =							VOCAB_FILES_NAMES
 __UpperCAmelCase		   =							PRETRAINED_VOCAB_FILES_MAP
 __UpperCAmelCase		   =							PRETRAINED_INIT_CONFIGURATION
 __UpperCAmelCase		   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __UpperCAmelCase		   =							LayoutLMTokenizer






 def __init__(self					,     lowerCAmelCase__=None					,     lowerCAmelCase__=None					,     lowerCAmelCase__=True					,     lowerCAmelCase__="[UNK]"					,     lowerCAmelCase__="[SEP]"					,     lowerCAmelCase__="[PAD]"					,     lowerCAmelCase__="[CLS]"					,     lowerCAmelCase__="[MASK]"					,     lowerCAmelCase__=True					,     lowerCAmelCase__=None					,     **lowerCAmelCase__					,     ):

  '''simple docstring'''







  super().__init__(
      lowerCAmelCase__					,     tokenizer_file=lowerCAmelCase__					,     do_lower_case=lowerCAmelCase__					,     unk_token=lowerCAmelCase__					,     sep_token=lowerCAmelCase__					,     pad_token=lowerCAmelCase__					,     cls_token=lowerCAmelCase__					,     mask_token=lowerCAmelCase__					,     tokenize_chinese_chars=lowerCAmelCase__					,     strip_accents=lowerCAmelCase__					,     **lowerCAmelCase__					,     )

  _UpperCamelCase      :	Optional[Any]					      =  json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
  if (
      normalizer_state.get("lowercase"					,     lowerCAmelCase__   ) != do_lower_case
      or normalizer_state.get("strip_accents"					,     lowerCAmelCase__   ) != strip_accents
      or normalizer_state.get("handle_chinese_chars"					,     lowerCAmelCase__   ) != tokenize_chinese_chars
  ):
   _UpperCamelCase      :	Union[str, Any]					      =  getattr(lowerCAmelCase__					,     normalizer_state.pop("type"   )   )
   _UpperCamelCase      :	int					      =  do_lower_case
   _UpperCamelCase      :	Any					      =  strip_accents
   _UpperCamelCase      :	Any					      =  tokenize_chinese_chars
   _UpperCamelCase      :	Dict					      =  normalizer_class(**lowerCAmelCase__   )

  _UpperCamelCase      :	Tuple					      =  do_lower_case






 def lowercase_      (self					,     lowerCAmelCase__					,     lowerCAmelCase__=None   ):

  '''simple docstring'''







  _UpperCamelCase      :	List[str]					      =  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

  if token_ids_a:
   output += token_ids_a + [self.sep_token_id]

  return output






 def lowercase_      (self					,     lowerCAmelCase__					,     lowerCAmelCase__ = None   ):

  '''simple docstring'''







  _UpperCamelCase      :	Dict					      =  [self.sep_token_id]
  _UpperCamelCase      :	Any					      =  [self.cls_token_id]
  if token_ids_a is None:
   return len(cls + token_ids_a + sep   ) * [0]
  return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






 def lowercase_      (self					,     lowerCAmelCase__					,     lowerCAmelCase__ = None   ):

  '''simple docstring'''







  _UpperCamelCase      :	int					      =  self._tokenizer.model.save(lowerCAmelCase__					,     name=lowerCAmelCase__   )
  return tuple(lowerCAmelCase__   )









"""simple docstring"""
import platform
from argparse import ArgumentParser

import huggingface_hub
import pandas
import pyarrow

from datasets import __version__ as version
from datasets.commands import BaseDatasetsCLICommand






def 				__lowerCAmelCase					(					__lowerCAmelCase			:		Union[str, Any]						)    ->       int:
 return EnvironmentCommand()







class  _SCREAMING_SNAKE_CASE       (     SCREAMING_SNAKE_CASE__				):



 '''simple docstring'''





 @staticmethod
 def lowercase_      (lowerCAmelCase__   ):

  '''simple docstring'''







  _UpperCamelCase      :	Optional[Any]					      =  parser.add_parser("env"					,     help="Print relevant system environment info."   )
  download_parser.set_defaults(func=lowerCAmelCase__   )






 def lowercase_      (self   ):

  '''simple docstring'''







  _UpperCamelCase      :	Any					      =  {
      "`datasets` version": version,
      "Platform": platform.platform(),
      "Python version": platform.python_version(),
      "Huggingface_hub version": huggingface_hub.__version__,
      "PyArrow version": pyarrow.__version__,
      "Pandas version": pandas.__version__,
  }

  print("\nCopy-and-paste the text below in your GitHub issue.\n"   )
  print(self.format_dict(lowerCAmelCase__   )   )

  return info






 @staticmethod
 def lowercase_      (lowerCAmelCase__   ):

  '''simple docstring'''







  return "\n".join([F"- {prop}: {val}" for prop, val in d.items()]   ) + "\n"



