

"""simple docstring"""
import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A									=						logging.get_logger(__name__)
A									=						{"""vocab_file""": """spiece.model"""}

A									=						{
    """vocab_file""": {
        """google/fnet-base""": """https://huggingface.co/google/fnet-base/resolve/main/spiece.model""",
        """google/fnet-large""": """https://huggingface.co/google/fnet-large/resolve/main/spiece.model""",
    },
}

A									=						{
    """google/fnet-base""": 512,
    """google/fnet-large""": 512,
}

A									=						"""▁"""

class        _UpperCamelCase	(						SCREAMING_SNAKE_CASE_	):


  """simple docstring"""





  snake_case_        =    VOCAB_FILES_NAMES
  snake_case_        =    PRETRAINED_VOCAB_FILES_MAP
  snake_case_        =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case_        =    ['input_ids', 'token_type_ids']



  def __init__( self					: Union[str, Any]       ,       snake_case					: Optional[int]       ,       snake_case					: int=False       ,       snake_case					: Optional[Any]=True       ,       snake_case					: int=True       ,       snake_case					: List[Any]="<unk>"       ,       snake_case					: Optional[Any]="[SEP]"       ,       snake_case					: Optional[Any]="<pad>"       ,       snake_case					: Union[str, Any]="[CLS]"       ,       snake_case					: str="[MASK]"       ,       snake_case					: Optional[Dict[str, Any]] = None       ,       **snake_case					: Dict       ,       )	->			None:


         '''simple docstring'''





         __magic_name__			:		Union[str, Any]        =     (
             AddedToken(lowercase_       ,       lstrip=lowercase_       ,       rstrip=lowercase_       ,       normalized=lowercase_      )
             if isinstance(lowercase_       ,       lowercase_      )
             else mask_token
         )

         __magic_name__			:		Optional[int]        =     {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=lowercase_       ,       remove_space=lowercase_       ,       keep_accents=lowercase_       ,       unk_token=lowercase_       ,       sep_token=lowercase_       ,       pad_token=lowercase_       ,       cls_token=lowercase_       ,       mask_token=lowercase_       ,       sp_model_kwargs=self.sp_model_kwargs       ,       **lowercase_       ,       )

         __magic_name__			:		Union[str, Any]        =     do_lower_case
         __magic_name__			:		int        =     remove_space
         __magic_name__			:		int        =     keep_accents
         __magic_name__			:		Tuple        =     vocab_file

         __magic_name__			:		Optional[Any]        =     spm.SentencePieceProcessor(**self.sp_model_kwargs      )
         self.sp_model.Load(lowercase_      )



  @property
  def 							_UpperCAmelCase      ( self					: Any      )	->			str:


         '''simple docstring'''





         return len(self.sp_model      )



  def 							_UpperCAmelCase      ( self					: Any      )	->			int:


         '''simple docstring'''





         __magic_name__			:		int        =     {self.convert_ids_to_tokens(lowercase_      ): i for i in range(self.vocab_size      )}
         vocab.update(self.added_tokens_encoder      )
         return vocab



  def __getstate__( self					: int      )	->			str:


         '''simple docstring'''





         __magic_name__			:		str        =     self.__dict__.copy()
         __magic_name__			:		Union[str, Any]        =     None
         return state



  def __setstate__( self					: str       ,       snake_case					: str      )	->			Any:


         '''simple docstring'''





         __magic_name__			:		Union[str, Any]        =     d

         # for backward compatibility
         if not hasattr(self       ,       '''sp_model_kwargs'''      ):
                __magic_name__			:		Union[str, Any]        =     {}

         __magic_name__			:		List[str]        =     spm.SentencePieceProcessor(**self.sp_model_kwargs      )
         self.sp_model.Load(self.vocab_file      )



  def 							_UpperCAmelCase      ( self					: str       ,       snake_case					: Tuple      )	->			Optional[int]:


         '''simple docstring'''





         if self.remove_space:
                __magic_name__			:		Optional[Any]        =     ''' '''.join(inputs.strip().split()      )
         else:
                __magic_name__			:		Union[str, Any]        =     inputs
         __magic_name__			:		Union[str, Any]        =     outputs.replace('''``'''       ,       '''"'''      ).replace('''\'\''''       ,       '''"'''      )

         if not self.keep_accents:
                __magic_name__			:		str        =     unicodedata.normalize('''NFKD'''       ,       lowercase_      )
                __magic_name__			:		Any        =     ''''''.join([c for c in outputs if not unicodedata.combining(lowercase_      )]      )
         if self.do_lower_case:
                __magic_name__			:		int        =     outputs.lower()

         return outputs



  def 							_UpperCAmelCase      ( self					: List[str]       ,       snake_case					: str      )	->			List[str]:


         '''simple docstring'''





         __magic_name__			:		List[str]        =     self.preprocess_text(lowercase_      )
         __magic_name__			:		Optional[Any]        =     self.sp_model.encode(lowercase_       ,       out_type=lowercase_      )
         __magic_name__			:		Optional[int]        =     []
         for piece in pieces:
                if len(lowercase_      ) > 1 and piece[-1] == str(''','''      ) and piece[-2].isdigit():
                       __magic_name__			:		List[Any]        =     self.sp_model.EncodeAsPieces(piece[:-1].replace(lowercase_       ,       ''''''      )      )
                       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                              if len(cur_pieces[0]      ) == 1:
                                     __magic_name__			:		str        =     cur_pieces[1:]
                              else:
                                     __magic_name__			:		List[Any]        =     cur_pieces[0][1:]
                       cur_pieces.append(piece[-1]      )
                       new_pieces.extend(lowercase_      )
                else:
                       new_pieces.append(lowercase_      )

         return new_pieces



  def 							_UpperCAmelCase      ( self					: str       ,       snake_case					: str      )	->			Union[str, Any]:


         '''simple docstring'''





         return self.sp_model.PieceToId(lowercase_      )



  def 							_UpperCAmelCase      ( self					: List[Any]       ,       snake_case					: Optional[Any]      )	->			Optional[int]:


         '''simple docstring'''





         return self.sp_model.IdToPiece(lowercase_      )



  def 							_UpperCAmelCase      ( self					: Dict       ,       snake_case					: Dict      )	->			Union[str, Any]:


         '''simple docstring'''





         __magic_name__			:		Tuple        =     []
         __magic_name__			:		Any        =     ''''''
         __magic_name__			:		Optional[Any]        =     False
         for token in tokens:
                # make sure that special tokens are not decoded using sentencepiece model
                if token in self.all_special_tokens:
                       if not prev_is_special:
                              out_string += " "
                       out_string += self.sp_model.decode(lowercase_      ) + token
                       __magic_name__			:		Dict        =     True
                       __magic_name__			:		str        =     []
                else:
                       current_sub_tokens.append(lowercase_      )
                       __magic_name__			:		Tuple        =     False
         out_string += self.sp_model.decode(lowercase_      )
         return out_string.strip()



  def 							_UpperCAmelCase      ( self					: List[Any]       ,       snake_case					: List[int]       ,       snake_case					: bool = False       ,       snake_case					: bool = None       ,       snake_case					: bool = True       ,       **snake_case					: Any       ,       )	->			str:


         '''simple docstring'''





         __magic_name__			:		Dict        =     kwargs.pop('''use_source_tokenizer'''       ,       lowercase_      )

         __magic_name__			:		int        =     self.convert_ids_to_tokens(lowercase_       ,       skip_special_tokens=lowercase_      )

         # To avoid mixing byte-level and unicode for byte-level BPT
         # we need to build string separately for added tokens and byte-level tokens
         # cf. https://github.com/huggingface/transformers/issues/1133
         __magic_name__			:		int        =     []
         __magic_name__			:		Union[str, Any]        =     []
         for token in filtered_tokens:
                if skip_special_tokens and token in self.all_special_ids:
                       continue
                if token in self.added_tokens_encoder:
                       if current_sub_text:
                              sub_texts.append(self.convert_tokens_to_string(lowercase_      )      )
                              __magic_name__			:		Optional[Any]        =     []
                       sub_texts.append(lowercase_      )
                else:
                       current_sub_text.append(lowercase_      )
         if current_sub_text:
                sub_texts.append(self.convert_tokens_to_string(lowercase_      )      )

         # Mimic the behavior of the Rust tokenizer:
         # No space after <unk>
         if spaces_between_special_tokens:
                __magic_name__			:		Optional[int]        =     re.sub(R'''(<unk>) '''       ,       R'''\1'''       ,       ''' '''.join(lowercase_      )      )
         else:
                __magic_name__			:		Optional[Any]        =     ''''''.join(lowercase_      )

         __magic_name__			:		str        =     (
             clean_up_tokenization_spaces
             if clean_up_tokenization_spaces is not None
             else self.clean_up_tokenization_spaces
         )
         if clean_up_tokenization_spaces:
                __magic_name__			:		Union[str, Any]        =     self.clean_up_tokenization(lowercase_      )
                return clean_text
         else:
                return text



  def 							_UpperCAmelCase      ( self					: Tuple       ,       snake_case					: List[int]       ,       snake_case					: Optional[List[int]] = None      )	->			List[int]:


         '''simple docstring'''





         __magic_name__			:		int        =     [self.sep_token_id]
         __magic_name__			:		Dict        =     [self.cls_token_id]
         if token_ids_a is None:
                return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep



  def 							_UpperCAmelCase      ( self					: Tuple       ,       snake_case					: List[int]       ,       snake_case					: Optional[List[int]] = None       ,       snake_case					: bool = False      )	->			List[int]:


         '''simple docstring'''





         if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=lowercase_       ,       token_ids_a=lowercase_       ,       already_has_special_tokens=lowercase_      )

         if token_ids_a is not None:
                return [1] + ([0] * len(lowercase_      )) + [1] + ([0] * len(lowercase_      )) + [1]
         return [1] + ([0] * len(lowercase_      )) + [1]



  def 							_UpperCAmelCase      ( self					: str       ,       snake_case					: List[int]       ,       snake_case					: Optional[List[int]] = None      )	->			List[int]:


         '''simple docstring'''





         __magic_name__			:		Tuple        =     [self.sep_token_id]
         __magic_name__			:		int        =     [self.cls_token_id]

         if token_ids_a is None:
                return len(cls + token_ids_a + sep      ) * [0]
         return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]




  def 							_UpperCAmelCase      ( self					: Optional[int]       ,       snake_case					: str       ,       snake_case					: Optional[str] = None      )	->			Tuple[str]:


         '''simple docstring'''





         if not os.path.isdir(lowercase_      ):
                logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""      )
                return
         __magic_name__			:		Tuple        =     os.path.join(
             lowercase_       ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

         if os.path.abspath(self.vocab_file      ) != os.path.abspath(lowercase_      ) and os.path.isfile(self.vocab_file      ):
                copyfile(self.vocab_file       ,       lowercase_      )
         elif not os.path.isfile(self.vocab_file      ):
                with open(lowercase_       ,       '''wb'''      ) as fi:
                       __magic_name__			:		Optional[Any]        =     self.sp_model.serialized_model_proto()
                       fi.write(lowercase_      )

         return (out_vocab_file,)





"""simple docstring"""


from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


snake_case_								=			logging.get_logger(__name__)

snake_case_								=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case_								=			{
    """vocab_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """openai/clip-vit-base-patch32""": (
            """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"""
        ),
    },
}

snake_case_								=			{
    """openai/clip-vit-base-patch32""": 77,
}







class  A_     (	SCREAMING_SNAKE_CASE_							):



						"""simple docstring"""
						__UpperCamelCase													=				VOCAB_FILES_NAMES
						__UpperCamelCase													=				PRETRAINED_VOCAB_FILES_MAP
						__UpperCamelCase													=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__UpperCamelCase													=				["""input_ids""", """attention_mask"""]
						__UpperCamelCase													=				CLIPTokenizer






						def __init__(   self					:int  ,    lowercase_					:Optional[int]=None  ,    lowercase_					:int=None  ,    lowercase_					:Optional[Any]=None  ,    lowercase_					:Dict="<|endoftext|>"  ,    lowercase_					:List[Any]="<|startoftext|>"  ,    lowercase_					:Any="<|endoftext|>"  ,    lowercase_					:List[str]="<|endoftext|>"  ,    **lowercase_					:Tuple  ,    )							->     List[Any]:
												super().__init__(
												    lowercase_  ,    lowercase_  ,    tokenizer_file=lowercase_  ,    unk_token=lowercase_  ,    bos_token=lowercase_  ,    eos_token=lowercase_  ,    pad_token=lowercase_  ,    **lowercase_  ,    )

												if not isinstance(self.backend_tokenizer.pre_tokenizer  ,    pre_tokenizers.Sequence   ):
																		raise ValueError(
																		    'The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'
																		    ' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'
																		    ' to be compatible with this version.The easiest way to do so is'
																		    ' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'
																		    ' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'
																		    ' transformers.'   )

												self._wrap_decode_method_backend_tokenizer()






						def       UpperCAmelCase__	(   self					:Dict   )							->     List[Any]:
												UpperCAmelCase			 =					self.backend_tokenizer.decode

												def new_decode_method(*lowercase_					:List[Any]  ,    **lowercase_					:Optional[Any]   ):
																		UpperCAmelCase			 =					orig_decode_method(*lowercase_  ,    **lowercase_   )
																		UpperCAmelCase			 =					text.replace(self.backend_tokenizer.model.end_of_word_suffix  ,    ' '   ).strip()
																		return text

												UpperCAmelCase			 =					new_decode_method






						def       UpperCAmelCase__	(   self					:Optional[int]  ,    lowercase_					:List[int]  ,    lowercase_					:Optional[List[int]] = None   )							->     List[int]:
												UpperCAmelCase			 =					[self.bos_token_id]
												UpperCAmelCase			 =					[self.eos_token_id]

												if token_ids_a is None:
																		return bos_token + token_ids_a + eos_token
												return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token






						def       UpperCAmelCase__	(   self					:Dict  ,    lowercase_					:List[int]  ,    lowercase_					:Optional[List[int]] = None   )							->     List[int]:
												UpperCAmelCase			 =					[self.bos_token_id]
												UpperCAmelCase			 =					[self.eos_token_id]

												if token_ids_a is None:
																		return len(bos_token + token_ids_a + eos_token   ) * [0]
												return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token   ) * [0]






						def       UpperCAmelCase__	(   self					:Optional[Any]  ,    lowercase_					:str  ,    lowercase_					:Optional[str] = None   )							->     Tuple[str]:
												UpperCAmelCase			 =					self._tokenizer.model.save(lowercase_  ,    name=lowercase_   )
												return tuple(lowercase_   )





