

import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer


class    SCREAMING_SNAKE_CASE__      (tf.keras.layers.Layer  ):





    def __init__(   self		,     a		,     a		,     a = None		,     a = None		,     a = None		,     a = "longest"		,     a = True		,     a = 512		,     a = None		,     a = True		,     a = True		,     a = True		,     **a		,     ):
         super().__init__()
         if use_fast_bert_tokenizer:
              lowercase__					:     Any   = FastBertTokenizer(
                  _lowerCAmelCase		,     token_out_type=tf.intaa		,     lower_case_nfd_strip_accents=_lowerCAmelCase		,     **_lowerCAmelCase)
         else:
              lowercase__					:     str   = tf.lookup.StaticVocabularyTable(
                  tf.lookup.KeyValueTensorInitializer(
                      keys=_lowerCAmelCase		,     key_dtype=tf.string		,     values=tf.range(tf.size(_lowerCAmelCase		,     out_type=tf.intaa)		,     dtype=tf.intaa)		,     value_dtype=tf.intaa		,     )		,     num_oov_buckets=1		,     )
              lowercase__					:     List[str]   = BertTokenizerLayer(
                  _lowerCAmelCase		,     token_out_type=tf.intaa		,     lower_case=_lowerCAmelCase		,     **_lowerCAmelCase)

         lowercase__					:     Optional[int]   = vocab_list
         lowercase__					:     List[str]   = do_lower_case
         lowercase__					:     int   = cls_token_id or vocab_list.index('[CLS]')
         lowercase__					:     Optional[int]   = sep_token_id or vocab_list.index('[SEP]')
         lowercase__					:     Dict   = pad_token_id or vocab_list.index('[PAD]')
         lowercase__					:     List[Any]   = ShrinkLongestTrimmer(max_length - 3		,     axis=1)  # Allow room for special tokens
         lowercase__					:     Optional[Any]   = max_length
         lowercase__					:     Optional[Any]   = padding
         lowercase__					:     Union[str, Any]   = truncation
         lowercase__					:     str   = pad_to_multiple_of
         lowercase__					:     str   = return_token_type_ids
         lowercase__					:     str   = return_attention_mask





    @classmethod
    def    snake_case_       (   cls		,     a		,     **a):  # noqa: F821
         lowercase__					:     Tuple   = kwargs.pop('do_lower_case'		,     _lowerCAmelCase)
         lowercase__					:     List[str]   = tokenizer.do_lower_case if do_lower_case is None else do_lower_case
         lowercase__					:     List[str]   = kwargs.pop('cls_token_id'		,     _lowerCAmelCase)
         lowercase__					:     Dict   = tokenizer.cls_token_id if cls_token_id is None else cls_token_id
         lowercase__					:     List[str]   = kwargs.pop('sep_token_id'		,     _lowerCAmelCase)
         lowercase__					:     int   = tokenizer.sep_token_id if sep_token_id is None else sep_token_id
         lowercase__					:     Tuple   = kwargs.pop('pad_token_id'		,     _lowerCAmelCase)
         lowercase__					:     Dict   = tokenizer.pad_token_id if pad_token_id is None else pad_token_id

         lowercase__					:     Dict   = tokenizer.get_vocab()
         lowercase__					:     str   = sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]		,     key=lambda a: x[1])
         lowercase__					:     Optional[Any]   = [entry[0] for entry in vocab]
         return cls(
             vocab_list=_lowerCAmelCase		,     do_lower_case=_lowerCAmelCase		,     cls_token_id=_lowerCAmelCase		,     sep_token_id=_lowerCAmelCase		,     pad_token_id=_lowerCAmelCase		,     **_lowerCAmelCase		,     )





    @classmethod
    def    snake_case_       (   cls		,     a		,     *a		,     **a):
         try:
              lowercase__					:     Dict   = BertTokenizer.from_pretrained(_lowerCAmelCase		,     *_lowerCAmelCase		,     **_lowerCAmelCase)
         except:  # noqa: E722
              from .tokenization_bert_fast import BertTokenizerFast

              lowercase__					:     Union[str, Any]   = BertTokenizerFast.from_pretrained(_lowerCAmelCase		,     *_lowerCAmelCase		,     **_lowerCAmelCase)
         return cls.from_tokenizer(_lowerCAmelCase		,     **_lowerCAmelCase)





    def    snake_case_       (   self		,     a):
         if self.do_lower_case:
              lowercase__					:     Any   = case_fold_utfa(_lowerCAmelCase)
         lowercase__					:     List[str]   = self.tf_tokenizer.tokenize(_lowerCAmelCase)
         return tokens.merge_dims(1		,     -1)





    def    snake_case_       (   self		,     a		,     a=None		,     a=None		,     a=None		,     a=None		,     a=None		,     a=None		,     a=None		,     ):
         if padding is None:
              lowercase__					:     List[Any]   = self.padding
         if padding not in ("longest", "max_length"):
              raise ValueError('Padding must be either \'longest\' or \'max_length\'!')
         if max_length is not None and text_pair is not None:
              # Because we have to instantiate a Trimmer to do it properly
              raise ValueError('max_length cannot be overridden at call time when truncating paired texts!')
         if max_length is None:
              lowercase__					:     Union[str, Any]   = self.max_length
         if truncation is None:
              lowercase__					:     str   = self.truncation
         if pad_to_multiple_of is None:
              lowercase__					:     Tuple   = self.pad_to_multiple_of
         if return_token_type_ids is None:
              lowercase__					:     Any   = self.return_token_type_ids
         if return_attention_mask is None:
              lowercase__					:     Tuple   = self.return_attention_mask
         if not isinstance(_lowerCAmelCase		,     tf.Tensor):
              lowercase__					:     Dict   = tf.convert_to_tensor(_lowerCAmelCase)
         if text_pair is not None and not isinstance(_lowerCAmelCase		,     tf.Tensor):
              lowercase__					:     List[str]   = tf.convert_to_tensor(_lowerCAmelCase)
         if text_pair is not None:
              if text.shape.rank > 1:
                   raise ValueError('text argument should not be multidimensional when a text pair is supplied!')
              if text_pair.shape.rank > 1:
                   raise ValueError('text_pair should not be multidimensional!')
         if text.shape.rank == 2:
              lowercase__   ,				lowercase__					:     Union[str, Any]   = text[:, 0], text[:, 1]
         lowercase__					:     Tuple   = self.unpaired_tokenize(_lowerCAmelCase)
         if text_pair is None:  # Unpaired text
              if truncation:
                   lowercase__					:     Union[str, Any]   = text[:, : max_length - 2]  # Allow room for special tokens
              lowercase__   ,				lowercase__					:     List[Any]   = combine_segments(
                  (text,)		,     start_of_sequence_id=self.cls_token_id		,     end_of_segment_id=self.sep_token_id)
         else:  # Paired text
              lowercase__					:     Union[str, Any]   = self.unpaired_tokenize(_lowerCAmelCase)
              if truncation:
                   lowercase__   ,				lowercase__					:     Dict   = self.paired_trimmer.trim([text, text_pair])
              lowercase__   ,				lowercase__					:     Any   = combine_segments(
                  (text, text_pair)		,     start_of_sequence_id=self.cls_token_id		,     end_of_segment_id=self.sep_token_id)
         if padding == "longest":
              lowercase__					:     List[Any]   = input_ids.bounding_shape(axis=1)
              if pad_to_multiple_of is not None:
                   # No ceiling division in tensorflow, so we negate floordiv instead
                   lowercase__					:     str   = pad_to_multiple_of * (-tf.math.floordiv(-pad_length		,     _lowerCAmelCase))
         else:
              lowercase__					:     Optional[Any]   = max_length

         lowercase__   ,				lowercase__					:     Union[str, Any]   = pad_model_inputs(_lowerCAmelCase		,     max_seq_length=_lowerCAmelCase		,     pad_value=self.pad_token_id)
         lowercase__					:     Union[str, Any]   = {'input_ids': input_ids}
         if return_attention_mask:
              lowercase__					:     str   = attention_mask
         if return_token_type_ids:
              lowercase__   ,				lowercase__					:     Any   = pad_model_inputs(
                  _lowerCAmelCase		,     max_seq_length=_lowerCAmelCase		,     pad_value=self.pad_token_id)
              lowercase__					:     str   = token_type_ids
         return output





    def    snake_case_       (   self):
         return {
             "vocab_list": self.vocab_list,
             "do_lower_case": self.do_lower_case,
             "cls_token_id": self.cls_token_id,
             "sep_token_id": self.sep_token_id,
             "pad_token_id": self.pad_token_id,
         }




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


snake_case_    						=				{
    '''configuration_donut_swin''': ['''DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''DonutSwinConfig'''],
    '''processing_donut''': ['''DonutProcessor'''],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case_    						=				[
           '''DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''DonutSwinModel''',
           '''DonutSwinPreTrainedModel''',
       ]

try:
       if not is_vision_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case_    						=				['''DonutFeatureExtractor''']
       snake_case_    						=				['''DonutImageProcessor''']


if TYPE_CHECKING:
       from .configuration_donut_swin import DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, DonutSwinConfig
       from .processing_donut import DonutProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_donut_swin import (
                  DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
                  DonutSwinModel,
                  DonutSwinPreTrainedModel,
              )

       try:
              if not is_vision_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .feature_extraction_donut import DonutFeatureExtractor
              from .image_processing_donut import DonutImageProcessor

else:
       import sys

       snake_case_    						=				_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

