






import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__            =	logging.get_logger(__name__)
UpperCAmelCase__            =	{"""vocab_file""": """spiece.model"""}

UpperCAmelCase__            =	{
    """vocab_file""": {
        """google/fnet-base""": """https://huggingface.co/google/fnet-base/resolve/main/spiece.model""",
        """google/fnet-large""": """https://huggingface.co/google/fnet-large/resolve/main/spiece.model""",
    },
}

UpperCAmelCase__            =	{
    """google/fnet-base""": 512,
    """google/fnet-large""": 512,
}

UpperCAmelCase__            =	"""▁"""






class 	lowercase_      (			_SCREAMING_SNAKE_CASE     ):






      '''simple docstring'''




      __snake_case  						=						VOCAB_FILES_NAMES
      __snake_case  						=						PRETRAINED_VOCAB_FILES_MAP
      __snake_case  						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __snake_case  						=						['''input_ids''', '''token_type_ids''']







      def __init__(   self   :		Any				,      __UpperCAmelCase   :		Union[str, Any]				,      __UpperCAmelCase   :		Union[str, Any]=False				,      __UpperCAmelCase   :		Optional[Any]=True				,      __UpperCAmelCase   :		str=True				,      __UpperCAmelCase   :		int="<unk>"				,      __UpperCAmelCase   :		List[Any]="[SEP]"				,      __UpperCAmelCase   :		List[Any]="<pad>"				,      __UpperCAmelCase   :		Optional[int]="[CLS]"				,      __UpperCAmelCase   :		Union[str, Any]="[MASK]"				,      __UpperCAmelCase   :		str = None				,      **__UpperCAmelCase   :		Tuple				,      )					->None:




       """simple docstring"""


       a							=   (
           AddedToken(__A				,      lstrip=__A				,      rstrip=__A				,      normalized=__A						)
           if isinstance(__A				,      __A						)
           else mask_token
       )

       a							=   {} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=__A				,      remove_space=__A				,      keep_accents=__A				,      unk_token=__A				,      sep_token=__A				,      pad_token=__A				,      cls_token=__A				,      mask_token=__A				,      sp_model_kwargs=self.sp_model_kwargs				,      **__A				,      )

       a							=   do_lower_case
       a							=   remove_space
       a							=   keep_accents
       a							=   vocab_file

       a							=   spm.SentencePieceProcessor(**self.sp_model_kwargs						)
       self.sp_model.Load(__A						)







      @property
      def    __lowerCAmelCase      (   self   :		int						)					->Optional[int]:




       """simple docstring"""


       return len(self.sp_model						)







      def    __lowerCAmelCase      (   self   :		str						)					->int:




       """simple docstring"""


       a							=   {self.convert_ids_to_tokens(__A						): i for i in range(self.vocab_size						)}
       vocab.update(self.added_tokens_encoder						)
       return vocab







      def __getstate__(   self   :		int						)					->List[Any]:




       """simple docstring"""


       a							=   self.__dict__.copy()
       a							=   None
       return state







      def __setstate__(   self   :		str				,      __UpperCAmelCase   :		List[Any]						)					->Any:




       """simple docstring"""


       a							=   d

       # for backward compatibility
       if not hasattr(self				,      '''sp_model_kwargs'''						):
        a							=   {}

       a							=   spm.SentencePieceProcessor(**self.sp_model_kwargs						)
       self.sp_model.Load(self.vocab_file						)







      def    __lowerCAmelCase      (   self   :		Optional[Any]				,      __UpperCAmelCase   :		str						)					->Optional[Any]:




       """simple docstring"""


       if self.remove_space:
        a							=   ''' '''.join(inputs.strip().split()						)
       else:
        a							=   inputs
       a							=   outputs.replace('''``'''				,      '''"'''						).replace('''\'\''''				,      '''"'''						)

       if not self.keep_accents:
        a							=   unicodedata.normalize('''NFKD'''				,      __A						)
        a							=   ''''''.join([c for c in outputs if not unicodedata.combining(__A						)]						)
       if self.do_lower_case:
        a							=   outputs.lower()

       return outputs







      def    __lowerCAmelCase      (   self   :		Optional[Any]				,      __UpperCAmelCase   :		List[Any]						)					->List[str]:




       """simple docstring"""


       a							=   self.preprocess_text(__A						)
       a							=   self.sp_model.encode(__A				,      out_type=__A						)
       a							=   []
       for piece in pieces:
        if len(__A						) > 1 and piece[-1] == str(''','''						) and piece[-2].isdigit():
         a							=   self.sp_model.EncodeAsPieces(piece[:-1].replace(__A				,      ''''''						)						)
         if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
          if len(cur_pieces[0]						) == 1:
           a							=   cur_pieces[1:]
          else:
           a							=   cur_pieces[0][1:]
         cur_pieces.append(piece[-1]						)
         new_pieces.extend(__A						)
        else:
         new_pieces.append(__A						)

       return new_pieces







      def    __lowerCAmelCase      (   self   :		List[str]				,      __UpperCAmelCase   :		Union[str, Any]						)					->Dict:




       """simple docstring"""


       return self.sp_model.PieceToId(__A						)







      def    __lowerCAmelCase      (   self   :		int				,      __UpperCAmelCase   :		Optional[int]						)					->Dict:




       """simple docstring"""


       return self.sp_model.IdToPiece(__A						)







      def    __lowerCAmelCase      (   self   :		Optional[int]				,      __UpperCAmelCase   :		List[Any]						)					->int:




       """simple docstring"""


       a							=   []
       a							=   ''''''
       a							=   False
       for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
         if not prev_is_special:
          out_string += " "
         out_string += self.sp_model.decode(__A						) + token
         a							=   True
         a							=   []
        else:
         current_sub_tokens.append(__A						)
         a							=   False
       out_string += self.sp_model.decode(__A						)
       return out_string.strip()







      def    __lowerCAmelCase      (   self   :		List[Any]				,      __UpperCAmelCase   :		List[Any]				,      __UpperCAmelCase   :		List[str] = False				,      __UpperCAmelCase   :		Any = None				,      __UpperCAmelCase   :		str = True				,      **__UpperCAmelCase   :		str				,      )					->str:




       """simple docstring"""


       a							=   kwargs.pop('''use_source_tokenizer'''				,      __A						)

       a							=   self.convert_ids_to_tokens(__A				,      skip_special_tokens=__A						)

       # To avoid mixing byte-level and unicode for byte-level BPT
       # we need to build string separately for added tokens and byte-level tokens
       # cf. https://github.com/huggingface/transformers/issues/1133
       a							=   []
       a							=   []
       for token in filtered_tokens:
        if skip_special_tokens and token in self.all_special_ids:
         continue
        if token in self.added_tokens_encoder:
         if current_sub_text:
          sub_texts.append(self.convert_tokens_to_string(__A						)						)
          a							=   []
         sub_texts.append(__A						)
        else:
         current_sub_text.append(__A						)
       if current_sub_text:
        sub_texts.append(self.convert_tokens_to_string(__A						)						)

       # Mimic the behavior of the Rust tokenizer:
       # No space after <unk>
       if spaces_between_special_tokens:
        a							=   re.sub(R'''(<unk>) '''				,      R'''\1'''				,      ''' '''.join(__A						)						)
       else:
        a							=   ''''''.join(__A						)

       a							=   (
           clean_up_tokenization_spaces
           if clean_up_tokenization_spaces is not None
           else self.clean_up_tokenization_spaces
       )
       if clean_up_tokenization_spaces:
        a							=   self.clean_up_tokenization(__A						)
        return clean_text
       else:
        return text







      def    __lowerCAmelCase      (   self   :		Any				,      __UpperCAmelCase   :		Union[str, Any]				,      __UpperCAmelCase   :		Optional[Any] = None						)					->List[int]:




       """simple docstring"""


       a							=   [self.sep_token_id]
       a							=   [self.cls_token_id]
       if token_ids_a is None:
        return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep







      def    __lowerCAmelCase      (   self   :		Any				,      __UpperCAmelCase   :		Any				,      __UpperCAmelCase   :		str = None				,      __UpperCAmelCase   :		int = False						)					->List[int]:




       """simple docstring"""


       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__A				,      token_ids_a=__A				,      already_has_special_tokens=__A						)

       if token_ids_a is not None:
        return [1] + ([0] * len(__A						)) + [1] + ([0] * len(__A						)) + [1]
       return [1] + ([0] * len(__A						)) + [1]







      def    __lowerCAmelCase      (   self   :		List[Any]				,      __UpperCAmelCase   :		List[Any]				,      __UpperCAmelCase   :		Optional[Any] = None						)					->List[int]:




       """simple docstring"""


       a							=   [self.sep_token_id]
       a							=   [self.cls_token_id]

       if token_ids_a is None:
        return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







      def    __lowerCAmelCase      (   self   :		Any				,      __UpperCAmelCase   :		Union[str, Any]				,      __UpperCAmelCase   :		int = None						)					->Tuple[str]:




       """simple docstring"""


       if not os.path.isdir(__A						):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
        return
       a							=   os.path.join(
           __A				,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(__A						) and os.path.isfile(self.vocab_file						):
        copyfile(self.vocab_file				,      __A						)
       elif not os.path.isfile(self.vocab_file						):
        with open(__A				,      '''wb'''						) as fi:
         a							=   self.sp_model.serialized_model_proto()
         fi.write(__A						)

       return (out_vocab_file,)










import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class 	lowercase_      (			lowercase     ):






      '''simple docstring'''




      __snake_case  						=						['''image_processor''', '''tokenizer''']
      __snake_case  						=						'''AutoImageProcessor'''
      __snake_case  						=						'''AutoTokenizer'''







      def __init__(   self   :		Any				,      __UpperCAmelCase   :		Union[str, Any]=None				,      __UpperCAmelCase   :		int=None				,      **__UpperCAmelCase   :		Union[str, Any]						)					->Dict:




       """simple docstring"""


       a							=   None
       if "feature_extractor" in kwargs:
        warnings.warn(
            '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
            ''' instead.'''				,      __UpperCAmelCase				,      )
        a							=   kwargs.pop('''feature_extractor'''						)

       a							=   image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
        raise ValueError('''You have to specify an image_processor.'''						)
       if tokenizer is None:
        raise ValueError('''You have to specify a tokenizer.'''						)

       super().__init__(__UpperCAmelCase				,      __UpperCAmelCase						)
       a							=   self.image_processor







      def __call__(   self   :		List[Any]				,      __UpperCAmelCase   :		Optional[Any]=None				,      __UpperCAmelCase   :		Optional[int]=None				,      __UpperCAmelCase   :		str=None				,      **__UpperCAmelCase   :		List[Any]						)					->Dict:




       """simple docstring"""


       if text is None and images is None:
        raise ValueError('''You have to specify either text or images. Both cannot be none.'''						)

       if text is not None:
        a							=   self.tokenizer(__UpperCAmelCase				,      return_tensors=__UpperCAmelCase				,      **__UpperCAmelCase						)

       if images is not None:
        a							=   self.image_processor(__UpperCAmelCase				,      return_tensors=__UpperCAmelCase				,      **__UpperCAmelCase						)

       if text is not None and images is not None:
        a							=   image_features.pixel_values
        return encoding
       elif text is not None:
        return encoding
       else:
        return BatchEncoding(data=dict(**__UpperCAmelCase						)				,      tensor_type=__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		List[Any]				,      *__UpperCAmelCase   :		Optional[Any]				,      **__UpperCAmelCase   :		List[str]						)					->Optional[Any]:




       """simple docstring"""


       return self.tokenizer.batch_decode(*__UpperCAmelCase				,      **__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		str				,      *__UpperCAmelCase   :		List[Any]				,      **__UpperCAmelCase   :		Optional[Any]						)					->Union[str, Any]:




       """simple docstring"""


       return self.tokenizer.decode(*__UpperCAmelCase				,      **__UpperCAmelCase						)







      @property
      def    __lowerCAmelCase      (   self   :		str						)					->str:




       """simple docstring"""


       a							=   self.tokenizer.model_input_names
       a							=   self.image_processor.model_input_names
       return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names						)						)







      @property
      def    __lowerCAmelCase      (   self   :		str						)					->Optional[Any]:




       """simple docstring"""


       warnings.warn(
           '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''				,      __UpperCAmelCase				,      )
       return self.image_processor_class







      @property
      def    __lowerCAmelCase      (   self   :		int						)					->Optional[Any]:




       """simple docstring"""


       warnings.warn(
           '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''				,      __UpperCAmelCase				,      )
       return self.image_processor


