





"""simple docstring"""




import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase_	 =logging.get_logger(__name__)
UpperCamelCase_	 ={"""vocab_file""": """spiece.model"""}

UpperCamelCase_	 ={
    """vocab_file""": {
        """google/fnet-base""": """https://huggingface.co/google/fnet-base/resolve/main/spiece.model""",
        """google/fnet-large""": """https://huggingface.co/google/fnet-large/resolve/main/spiece.model""",
    },
}

UpperCamelCase_	 ={
    """google/fnet-base""": 512,
    """google/fnet-large""": 512,
}

UpperCamelCase_	 ="""▁"""




class 					_a		( _lowerCAmelCase							):
				UpperCamelCase								=       VOCAB_FILES_NAMES
				UpperCamelCase								=       PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase								=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase								=       ['''input_ids''', '''token_type_ids''']






				def __init__(					self						:       Union[str, Any],					lowerCAmelCase__						:       Dict,					lowerCAmelCase__						:       int=False,					lowerCAmelCase__						:       Tuple=True,					lowerCAmelCase__						:       List[str]=True,					lowerCAmelCase__						:       Tuple="<unk>",					lowerCAmelCase__						:       List[Any]="[SEP]",					lowerCAmelCase__						:       int="<pad>",					lowerCAmelCase__						:       int="[CLS]",					lowerCAmelCase__						:       int="[MASK]",					lowerCAmelCase__						:       Optional[Dict[str, Any]] = None,					**lowerCAmelCase__						:       Union[str, Any],					)      -> None:




										'''simple docstring'''



										_UpperCamelCase     :   Dict      =       (
										    AddedToken(lowerCAmelCase__,					lstrip=lowerCAmelCase__,					rstrip=lowerCAmelCase__,					normalized=lowerCAmelCase__				)
										    if isinstance(lowerCAmelCase__,					lowerCAmelCase__				)
										    else mask_token
										)

										_UpperCamelCase     :   Any      =       {} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=lowerCAmelCase__,					remove_space=lowerCAmelCase__,					keep_accents=lowerCAmelCase__,					unk_token=lowerCAmelCase__,					sep_token=lowerCAmelCase__,					pad_token=lowerCAmelCase__,					cls_token=lowerCAmelCase__,					mask_token=lowerCAmelCase__,					sp_model_kwargs=self.sp_model_kwargs,					**lowerCAmelCase__,					)

										_UpperCamelCase     :   List[Any]      =       do_lower_case
										_UpperCamelCase     :   Tuple      =       remove_space
										_UpperCamelCase     :   Dict      =       keep_accents
										_UpperCamelCase     :   Dict      =       vocab_file

										_UpperCamelCase     :   str      =       spm.SentencePieceProcessor(**self.sp_model_kwargs				)
										self.sp_model.Load(lowerCAmelCase__				)






				@property
				def 							snake_case      (					self						:       str				)      -> Tuple:




										'''simple docstring'''



										return len(self.sp_model				)






				def 							snake_case      (					self						:       Dict				)      -> str:




										'''simple docstring'''



										_UpperCamelCase     :   Dict      =       {self.convert_ids_to_tokens(lowerCAmelCase__				): i for i in range(self.vocab_size				)}
										vocab.update(self.added_tokens_encoder				)
										return vocab






				def __getstate__(					self						:       Any				)      -> int:




										'''simple docstring'''



										_UpperCamelCase     :   List[str]      =       self.__dict__.copy()
										_UpperCamelCase     :   Optional[Any]      =       None
										return state






				def __setstate__(					self						:       Optional[Any],					lowerCAmelCase__						:       Union[str, Any]				)      -> Dict:




										'''simple docstring'''



										_UpperCamelCase     :   Dict      =       d

										# for backward compatibility
										if not hasattr(self,					'''sp_model_kwargs'''				):
																_UpperCamelCase     :   Optional[Any]      =       {}

										_UpperCamelCase     :   int      =       spm.SentencePieceProcessor(**self.sp_model_kwargs				)
										self.sp_model.Load(self.vocab_file				)






				def 							snake_case      (					self						:       List[str],					lowerCAmelCase__						:       Optional[Any]				)      -> Dict:




										'''simple docstring'''



										if self.remove_space:
																_UpperCamelCase     :   Any      =       ''' '''.join(inputs.strip().split()				)
										else:
																_UpperCamelCase     :   int      =       inputs
										_UpperCamelCase     :   Optional[Any]      =       outputs.replace('''``''',					'''"'''				).replace('''\'\'''',					'''"'''				)

										if not self.keep_accents:
																_UpperCamelCase     :   Union[str, Any]      =       unicodedata.normalize('''NFKD''',					lowerCAmelCase__				)
																_UpperCamelCase     :   Dict      =       ''''''.join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__				)]				)
										if self.do_lower_case:
																_UpperCamelCase     :   Dict      =       outputs.lower()

										return outputs






				def 							snake_case      (					self						:       Union[str, Any],					lowerCAmelCase__						:       str				)      -> List[str]:




										'''simple docstring'''



										_UpperCamelCase     :   Any      =       self.preprocess_text(lowerCAmelCase__				)
										_UpperCamelCase     :   Tuple      =       self.sp_model.encode(lowerCAmelCase__,					out_type=lowerCAmelCase__				)
										_UpperCamelCase     :   int      =       []
										for piece in pieces:
																if len(lowerCAmelCase__				) > 1 and piece[-1] == str(''','''				) and piece[-2].isdigit():
																						_UpperCamelCase     :   Optional[Any]      =       self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__,					''''''				)				)
																						if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																												if len(cur_pieces[0]				) == 1:
																																		_UpperCamelCase     :   Optional[int]      =       cur_pieces[1:]
																												else:
																																		_UpperCamelCase     :   Optional[int]      =       cur_pieces[0][1:]
																						cur_pieces.append(piece[-1]				)
																						new_pieces.extend(lowerCAmelCase__				)
																else:
																						new_pieces.append(lowerCAmelCase__				)

										return new_pieces






				def 							snake_case      (					self						:       Dict,					lowerCAmelCase__						:       Tuple				)      -> Tuple:




										'''simple docstring'''



										return self.sp_model.PieceToId(lowerCAmelCase__				)






				def 							snake_case      (					self						:       Union[str, Any],					lowerCAmelCase__						:       List[str]				)      -> Dict:




										'''simple docstring'''



										return self.sp_model.IdToPiece(lowerCAmelCase__				)






				def 							snake_case      (					self						:       Any,					lowerCAmelCase__						:       Union[str, Any]				)      -> Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase     :   Any      =       []
										_UpperCamelCase     :   Optional[int]      =       ''''''
										_UpperCamelCase     :   Optional[int]      =       False
										for token in tokens:
																# make sure that special tokens are not decoded using sentencepiece model
																if token in self.all_special_tokens:
																						if not prev_is_special:
																												out_string += " "
																						out_string += self.sp_model.decode(lowerCAmelCase__				) + token
																						_UpperCamelCase     :   Optional[int]      =       True
																						_UpperCamelCase     :   Union[str, Any]      =       []
																else:
																						current_sub_tokens.append(lowerCAmelCase__				)
																						_UpperCamelCase     :   int      =       False
										out_string += self.sp_model.decode(lowerCAmelCase__				)
										return out_string.strip()






				def 							snake_case      (					self						:       str,					lowerCAmelCase__						:       List[int],					lowerCAmelCase__						:       bool = False,					lowerCAmelCase__						:       bool = None,					lowerCAmelCase__						:       bool = True,					**lowerCAmelCase__						:       str,					)      -> str:




										'''simple docstring'''



										_UpperCamelCase     :   int      =       kwargs.pop('''use_source_tokenizer''',					lowerCAmelCase__				)

										_UpperCamelCase     :   Dict      =       self.convert_ids_to_tokens(lowerCAmelCase__,					skip_special_tokens=lowerCAmelCase__				)

										# To avoid mixing byte-level and unicode for byte-level BPT
										# we need to build string separately for added tokens and byte-level tokens
										# cf. https://github.com/huggingface/transformers/issues/1133
										_UpperCamelCase     :   str      =       []
										_UpperCamelCase     :   str      =       []
										for token in filtered_tokens:
																if skip_special_tokens and token in self.all_special_ids:
																						continue
																if token in self.added_tokens_encoder:
																						if current_sub_text:
																												sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__				)				)
																												_UpperCamelCase     :   str      =       []
																						sub_texts.append(lowerCAmelCase__				)
																else:
																						current_sub_text.append(lowerCAmelCase__				)
										if current_sub_text:
																sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__				)				)

										# Mimic the behavior of the Rust tokenizer:
										# No space after <unk>
										if spaces_between_special_tokens:
																_UpperCamelCase     :   Optional[int]      =       re.sub(R'''(<unk>) ''',					R'''\1''',					''' '''.join(lowerCAmelCase__				)				)
										else:
																_UpperCamelCase     :   int      =       ''''''.join(lowerCAmelCase__				)

										_UpperCamelCase     :   Optional[Any]      =       (
										    clean_up_tokenization_spaces
										    if clean_up_tokenization_spaces is not None
										    else self.clean_up_tokenization_spaces
										)
										if clean_up_tokenization_spaces:
																_UpperCamelCase     :   str      =       self.clean_up_tokenization(lowerCAmelCase__				)
																return clean_text
										else:
																return text






				def 							snake_case      (					self						:       Tuple,					lowerCAmelCase__						:       List[int],					lowerCAmelCase__						:       Optional[List[int]] = None				)      -> List[int]:




										'''simple docstring'''



										_UpperCamelCase     :   str      =       [self.sep_token_id]
										_UpperCamelCase     :   Any      =       [self.cls_token_id]
										if token_ids_a is None:
																return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep






				def 							snake_case      (					self						:       Optional[Any],					lowerCAmelCase__						:       List[int],					lowerCAmelCase__						:       Optional[List[int]] = None,					lowerCAmelCase__						:       bool = False				)      -> List[int]:




										'''simple docstring'''



										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=lowerCAmelCase__,					token_ids_a=lowerCAmelCase__,					already_has_special_tokens=lowerCAmelCase__				)

										if token_ids_a is not None:
																return [1] + ([0] * len(lowerCAmelCase__				)) + [1] + ([0] * len(lowerCAmelCase__				)) + [1]
										return [1] + ([0] * len(lowerCAmelCase__				)) + [1]






				def 							snake_case      (					self						:       str,					lowerCAmelCase__						:       List[int],					lowerCAmelCase__						:       Optional[List[int]] = None				)      -> List[int]:




										'''simple docstring'''



										_UpperCamelCase     :   int      =       [self.sep_token_id]
										_UpperCamelCase     :   Union[str, Any]      =       [self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep				) * [0]
										return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






				def 							snake_case      (					self						:       Any,					lowerCAmelCase__						:       str,					lowerCAmelCase__						:       Optional[str] = None				)      -> Tuple[str]:




										'''simple docstring'''



										if not os.path.isdir(lowerCAmelCase__				):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
																return
										_UpperCamelCase     :   Any      =       os.path.join(
										    lowerCAmelCase__,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

										if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCAmelCase__				) and os.path.isfile(self.vocab_file				):
																copyfile(self.vocab_file,					lowerCAmelCase__				)
										elif not os.path.isfile(self.vocab_file				):
																with open(lowerCAmelCase__,					'''wb'''				) as fi:
																						_UpperCamelCase     :   Optional[int]      =       self.sp_model.serialized_model_proto()
																						fi.write(lowerCAmelCase__				)

										return (out_vocab_file,)




"""simple docstring"""






from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
						import PIL


lowerCAmelCase__    :    Tuple			=      logging.get_logger(__name__)
class 							snake_case			(					__UpperCAmelCase							):




					"""simple docstring"""

					snake_case__							       =					["pixel_values"]







					def __init__(							self  :      str       ,lowerCamelCase__  :      bool = True       ,lowerCamelCase__  :      Dict[str, int] = None       ,lowerCamelCase__  :      bool = True       ,lowerCamelCase__  :      Dict[str, int] = None       ,lowerCamelCase__  :      PILImageResampling = PILImageResampling.BICUBIC       ,lowerCamelCase__  :      bool = True       ,lowerCamelCase__  :      Union[int, float] = 1 / 255       ,lowerCamelCase__  :      bool = True       ,lowerCamelCase__  :      Optional[Union[float, List[float]]] = None       ,lowerCamelCase__  :      Optional[Union[float, List[float]]] = None       ,**lowerCamelCase__  :      List[Any]       ,):
									super().__init__(**lowerCamelCase__     )
									UpperCAmelCase__			 =				crop_size if crop_size is not None else {'height': 256, 'width': 256}
									UpperCAmelCase__			 =				get_size_dict(lowerCamelCase__       ,param_name='crop_size'     )
									UpperCAmelCase__			 =				size if size is not None else {'height': 224, 'width': 224}
									UpperCAmelCase__			 =				get_size_dict(lowerCamelCase__     )

									UpperCAmelCase__			 =				do_center_crop
									UpperCAmelCase__			 =				crop_size
									UpperCAmelCase__			 =				do_resize
									UpperCAmelCase__			 =				size
									UpperCAmelCase__			 =				resample
									UpperCAmelCase__			 =				do_rescale
									UpperCAmelCase__			 =				rescale_factor
									UpperCAmelCase__			 =				do_normalize
									UpperCAmelCase__			 =				image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
									UpperCAmelCase__			 =				image_std if image_std is not None else IMAGENET_DEFAULT_STD







					def 						__lowerCAmelCase		(							self  :      Optional[int]       ,lowerCamelCase__  :      np.ndarray       ,lowerCamelCase__  :      Dict[str, int]       ,lowerCamelCase__  :      Optional[int] = None       ,lowerCamelCase__  :      Optional[Union[str, ChannelDimension]] = None       ,**lowerCamelCase__  :      List[str]       ,):
									UpperCAmelCase__			 =				self.size if size is None else size
									UpperCAmelCase__			 =				get_size_dict(lowerCamelCase__     )
									UpperCAmelCase__			 =				get_size_dict(lowerCamelCase__       ,param_name='crop_size'     )

									UpperCAmelCase__ ,      UpperCAmelCase__			 =				get_image_size(lowerCamelCase__     )
									UpperCAmelCase__			 =				min(lowerCamelCase__       ,lowerCamelCase__     )
									UpperCAmelCase__			 =				(size['height'] / crop_size['height']) * min_dim
									UpperCAmelCase__			 =				(size['width'] / crop_size['width']) * min_dim
									return center_crop(lowerCamelCase__       ,size=(cropped_height, cropped_width)       ,data_format=lowerCamelCase__       ,**lowerCamelCase__     )







					def 						__lowerCAmelCase		(							self  :      Any       ,lowerCamelCase__  :      np.ndarray       ,lowerCamelCase__  :      Dict[str, int]       ,lowerCamelCase__  :      PILImageResampling = PIL.Image.BICUBIC       ,lowerCamelCase__  :      Optional[Union[str, ChannelDimension]] = None       ,**lowerCamelCase__  :      Any       ,):
									UpperCAmelCase__			 =				get_size_dict(lowerCamelCase__     )
									if "height" not in size or "width" not in size:
													raise ValueError(f'''The size dictionary must contain the keys \'height\' and \'width\'. Got {size.keys()}'''     )
									return resize(
									    lowerCamelCase__       ,size=(size['height'], size['width'])       ,resample=lowerCamelCase__       ,data_format=lowerCamelCase__       ,**lowerCamelCase__     )







					def 						__lowerCAmelCase		(							self  :      Optional[int]       ,lowerCamelCase__  :      np.ndarray       ,lowerCamelCase__  :      Union[int, float]       ,lowerCamelCase__  :      Optional[Union[str, ChannelDimension]] = None       ,**lowerCamelCase__  :      int       ,):
									return rescale(lowerCamelCase__       ,scale=lowerCamelCase__       ,data_format=lowerCamelCase__       ,**lowerCamelCase__     )







					def 						__lowerCAmelCase		(							self  :      int       ,lowerCamelCase__  :      np.ndarray       ,lowerCamelCase__  :      Union[float, List[float]]       ,lowerCamelCase__  :      Union[float, List[float]]       ,lowerCamelCase__  :      Optional[Union[str, ChannelDimension]] = None       ,**lowerCamelCase__  :      List[Any]       ,):
									return normalize(lowerCamelCase__       ,mean=lowerCamelCase__       ,std=lowerCamelCase__       ,data_format=lowerCamelCase__       ,**lowerCamelCase__     )







					def 						__lowerCAmelCase		(							self  :      Optional[Any]       ,lowerCamelCase__  :      ImageInput       ,lowerCamelCase__  :      Optional[bool] = None       ,lowerCamelCase__  :      Optional[Dict[str, int]] = None       ,lowerCamelCase__  :      Optional[bool] = None       ,lowerCamelCase__  :      Optional[Dict[str, int]] = None       ,lowerCamelCase__  :      PILImageResampling = None       ,lowerCamelCase__  :      Optional[bool] = None       ,lowerCamelCase__  :      Optional[float] = None       ,lowerCamelCase__  :      Optional[bool] = None       ,lowerCamelCase__  :      Optional[Union[float, List[float]]] = None       ,lowerCamelCase__  :      Optional[Union[float, List[float]]] = None       ,lowerCamelCase__  :      Optional[Union[str, TensorType]] = None       ,lowerCamelCase__  :      ChannelDimension = ChannelDimension.FIRST       ,**lowerCamelCase__  :      Optional[int]       ,):
									UpperCAmelCase__			 =				do_center_crop if do_center_crop is not None else self.do_center_crop
									UpperCAmelCase__			 =				crop_size if crop_size is not None else self.crop_size
									UpperCAmelCase__			 =				get_size_dict(lowerCamelCase__       ,param_name='crop_size'     )
									UpperCAmelCase__			 =				do_resize if do_resize is not None else self.do_resize
									UpperCAmelCase__			 =				size if size is not None else self.size
									UpperCAmelCase__			 =				get_size_dict(lowerCamelCase__     )
									UpperCAmelCase__			 =				resample if resample is not None else self.resample
									UpperCAmelCase__			 =				do_rescale if do_rescale is not None else self.do_rescale
									UpperCAmelCase__			 =				rescale_factor if rescale_factor is not None else self.rescale_factor
									UpperCAmelCase__			 =				do_normalize if do_normalize is not None else self.do_normalize
									UpperCAmelCase__			 =				image_mean if image_mean is not None else self.image_mean
									UpperCAmelCase__			 =				image_std if image_std is not None else self.image_std

									UpperCAmelCase__			 =				make_list_of_images(lowerCamelCase__     )

									if not valid_images(lowerCamelCase__     ):
													raise ValueError(
													    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
													    'torch.Tensor, tf.Tensor or jax.ndarray.'     )

									if do_center_crop and crop_size is None:
													raise ValueError('If `do_center_crop` is set to `True`, `crop_size` must be provided.'     )

									if do_resize and size is None:
													raise ValueError('Size must be specified if do_resize is True.'     )

									if do_rescale and rescale_factor is None:
													raise ValueError('Rescale factor must be specified if do_rescale is True.'     )

									if do_normalize and (image_mean is None or image_std is None):
													raise ValueError('Image mean and image standard deviation must be specified if do_normalize is True.'     )

									# All transformations expect numpy arrays.
									UpperCAmelCase__			 =				[to_numpy_array(lowerCamelCase__     ) for image in images]

									if do_center_crop:
													UpperCAmelCase__			 =				[self.center_crop(lowerCamelCase__       ,lowerCamelCase__       ,size=lowerCamelCase__     ) for image in images]

									if do_resize:
													UpperCAmelCase__			 =				[self.resize(image=lowerCamelCase__       ,size=lowerCamelCase__       ,resample=lowerCamelCase__     ) for image in images]

									if do_rescale:
													UpperCAmelCase__			 =				[self.rescale(image=lowerCamelCase__       ,scale=lowerCamelCase__     ) for image in images]

									if do_normalize:
													UpperCAmelCase__			 =				[self.normalize(image=lowerCamelCase__       ,mean=lowerCamelCase__       ,std=lowerCamelCase__     ) for image in images]

									UpperCAmelCase__			 =				[to_channel_dimension_format(lowerCamelCase__       ,lowerCamelCase__     ) for image in images]

									UpperCAmelCase__			 =				{'pixel_values': images}
									return BatchFeature(data=lowerCamelCase__       ,tensor_type=lowerCamelCase__     )



