


'''simple docstring'''




import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowercase  :      Optional[int]      		=				logging.get_logger(__name__)
__lowercase  :      Optional[Any]      		=				{"""vocab_file""": """spiece.model"""}

__lowercase  :      int      		=				{
    """vocab_file""": {
        """google/fnet-base""": """https://huggingface.co/google/fnet-base/resolve/main/spiece.model""",
        """google/fnet-large""": """https://huggingface.co/google/fnet-large/resolve/main/spiece.model""",
    },
}

__lowercase  :      Union[str, Any]      		=				{
    """google/fnet-base""": 5_12,
    """google/fnet-large""": 5_12,
}

__lowercase  :      Optional[Any]      		=				"""▁"""





class _snake_case   ( a__   ):
							"""simple docstring"""





							__a												=     VOCAB_FILES_NAMES
							__a												=     PRETRAINED_VOCAB_FILES_MAP
							__a												=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a												=     ['''input_ids''', '''token_type_ids''']







							def __init__(   self			,				__A			,				__A=False			,				__A=True			,				__A=True			,				__A="<unk>"			,				__A="[SEP]"			,				__A="<pad>"			,				__A="[CLS]"			,				__A="[MASK]"			,				__A = None			,				**__A			,				)     ->None:



											"""simple docstring"""







											__lowerCamelCase		:      Dict											=					(
											    AddedToken(__A			,				lstrip=__A			,				rstrip=__A			,				normalized=__A	)
											    if isinstance(__A			,				__A	)
											    else mask_token
											)

											__lowerCamelCase		:      List[Any]											=					{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=__A			,				remove_space=__A			,				keep_accents=__A			,				unk_token=__A			,				sep_token=__A			,				pad_token=__A			,				cls_token=__A			,				mask_token=__A			,				sp_model_kwargs=self.sp_model_kwargs			,				**__A			,				)

											__lowerCamelCase		:      Optional[int]											=					do_lower_case
											__lowerCamelCase		:      str											=					remove_space
											__lowerCamelCase		:      int											=					keep_accents
											__lowerCamelCase		:      Tuple											=					vocab_file

											__lowerCamelCase		:      Tuple											=					spm.SentencePieceProcessor(**self.sp_model_kwargs	)
											self.sp_model.Load(__A	)







							@property
							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:



											"""simple docstring"""







											return len(self.sp_model	)







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:



											"""simple docstring"""







											__lowerCamelCase		:      Dict											=					{self.convert_ids_to_tokens(__A	): i for i in range(self.vocab_size	)}
											vocab.update(self.added_tokens_encoder	)
											return vocab







							def __getstate__(   self	)     ->Optional[Any]:



											"""simple docstring"""







											__lowerCamelCase		:      int											=					self.__dict__.copy()
											__lowerCamelCase		:      int											=					None
											return state







							def __setstate__(   self			,				__A	)     ->Dict:



											"""simple docstring"""







											__lowerCamelCase		:      int											=					d

											# for backward compatibility
											if not hasattr(self			,				"sp_model_kwargs"	):
															__lowerCamelCase		:      int											=					{}

											__lowerCamelCase		:      Tuple											=					spm.SentencePieceProcessor(**self.sp_model_kwargs	)
											self.sp_model.Load(self.vocab_file	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->int:



											"""simple docstring"""







											if self.remove_space:
															__lowerCamelCase		:      Optional[Any]											=					" ".join(inputs.strip().split()	)
											else:
															__lowerCamelCase		:      Tuple											=					inputs
											__lowerCamelCase		:      List[Any]											=					outputs.replace("``"			,				"\""	).replace("''"			,				"\""	)

											if not self.keep_accents:
															__lowerCamelCase		:      str											=					unicodedata.normalize("NFKD"			,				__A	)
															__lowerCamelCase		:      str											=					"".join([c for c in outputs if not unicodedata.combining(__A	)]	)
											if self.do_lower_case:
															__lowerCamelCase		:      int											=					outputs.lower()

											return outputs







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->List[str]:



											"""simple docstring"""







											__lowerCamelCase		:      List[Any]											=					self.preprocess_text(__A	)
											__lowerCamelCase		:      int											=					self.sp_model.encode(__A			,				out_type=__A	)
											__lowerCamelCase		:      Union[str, Any]											=					[]
											for piece in pieces:
															if len(__A	) > 1 and piece[-1] == str(","	) and piece[-2].isdigit():
																			__lowerCamelCase		:      int											=					self.sp_model.EncodeAsPieces(piece[:-1].replace(__A			,				""	)	)
																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																							if len(cur_pieces[0]	) == 1:
																											__lowerCamelCase		:      List[str]											=					cur_pieces[1:]
																							else:
																											__lowerCamelCase		:      Any											=					cur_pieces[0][1:]
																			cur_pieces.append(piece[-1]	)
																			new_pieces.extend(__A	)
															else:
																			new_pieces.append(__A	)

											return new_pieces







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->Dict:



											"""simple docstring"""







											return self.sp_model.PieceToId(__A	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->Optional[Any]:



											"""simple docstring"""







											return self.sp_model.IdToPiece(__A	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->Optional[Any]:



											"""simple docstring"""







											__lowerCamelCase		:      Any											=					[]
											__lowerCamelCase		:      Union[str, Any]											=					""
											__lowerCamelCase		:      int											=					False
											for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																			if not prev_is_special:
																							out_string += " "
																			out_string += self.sp_model.decode(__A	) + token
																			__lowerCamelCase		:      Tuple											=					True
																			__lowerCamelCase		:      str											=					[]
															else:
																			current_sub_tokens.append(__A	)
																			__lowerCamelCase		:      List[str]											=					False
											out_string += self.sp_model.decode(__A	)
											return out_string.strip()







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = False			,				__A = None			,				__A = True			,				**__A			,				)     ->str:



											"""simple docstring"""







											__lowerCamelCase		:      Dict											=					kwargs.pop("use_source_tokenizer"			,				__A	)

											__lowerCamelCase		:      int											=					self.convert_ids_to_tokens(__A			,				skip_special_tokens=__A	)

											# To avoid mixing byte-level and unicode for byte-level BPT
											# we need to build string separately for added tokens and byte-level tokens
											# cf. https://github.com/huggingface/transformers/issues/1133
											__lowerCamelCase		:      int											=					[]
											__lowerCamelCase		:      Optional[int]											=					[]
											for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																			continue
															if token in self.added_tokens_encoder:
																			if current_sub_text:
																							sub_texts.append(self.convert_tokens_to_string(__A	)	)
																							__lowerCamelCase		:      List[str]											=					[]
																			sub_texts.append(__A	)
															else:
																			current_sub_text.append(__A	)
											if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(__A	)	)

											# Mimic the behavior of the Rust tokenizer:
											# No space after <unk>
											if spaces_between_special_tokens:
															__lowerCamelCase		:      Optional[int]											=					re.sub(r"(<unk>) "			,				r"\1"			,				" ".join(__A	)	)
											else:
															__lowerCamelCase		:      Optional[Any]											=					"".join(__A	)

											__lowerCamelCase		:      int											=					(
											    clean_up_tokenization_spaces
											    if clean_up_tokenization_spaces is not None
											    else self.clean_up_tokenization_spaces
											)
											if clean_up_tokenization_spaces:
															__lowerCamelCase		:      Tuple											=					self.clean_up_tokenization(__A	)
															return clean_text
											else:
															return text







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None	)     ->List[int]:



											"""simple docstring"""







											__lowerCamelCase		:      str											=					[self.sep_token_id]
											__lowerCamelCase		:      int											=					[self.cls_token_id]
											if token_ids_a is None:
															return cls + token_ids_a + sep
											return cls + token_ids_a + sep + token_ids_a + sep







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None			,				__A = False	)     ->List[int]:



											"""simple docstring"""







											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=__A			,				token_ids_a=__A			,				already_has_special_tokens=__A	)

											if token_ids_a is not None:
															return [1] + ([0] * len(__A	)) + [1] + ([0] * len(__A	)) + [1]
											return [1] + ([0] * len(__A	)) + [1]







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None	)     ->List[int]:



											"""simple docstring"""







											__lowerCamelCase		:      Tuple											=					[self.sep_token_id]
											__lowerCamelCase		:      Optional[int]											=					[self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep	) * [0]
											return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None	)     ->Tuple[str]:



											"""simple docstring"""







											if not os.path.isdir(__A	):
															logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
															return
											__lowerCamelCase		:      Optional[Any]											=					os.path.join(
											    __A			,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)

											if os.path.abspath(self.vocab_file	) != os.path.abspath(__A	) and os.path.isfile(self.vocab_file	):
															copyfile(self.vocab_file			,				__A	)
											elif not os.path.isfile(self.vocab_file	):
															with open(__A			,				"wb"	) as fi:
																			__lowerCamelCase		:      Tuple											=					self.sp_model.serialized_model_proto()
																			fi.write(__A	)

											return (out_vocab_file,)









'''simple docstring'''




from __future__ import annotations

from collections import deque
from enum import Enum
from math import atana, degrees
from sys import maxsize





class _snake_case   ( a__   ):
							"""simple docstring"""





							__a												=     1
							__a												=     2
							__a												=     3







							def __repr__(   self	)     ->Tuple:



											"""simple docstring"""







											return f'{self.__class__.__name__}.{self.name}'





def 							a_       (						lowercase							,						lowercase							,						lowercase    )			->float:






				'''simple docstring'''

				__lowerCamelCase						,					__lowerCamelCase		:      str											=					point
				return degrees(atana(y - miny							,						x - minx    )    )





def 							a_       (						lowercase							,						lowercase							,						lowercase    )			->Direction:






				'''simple docstring'''

				__lowerCamelCase						,					__lowerCamelCase		:      List[str]											=					starting
				__lowerCamelCase						,					__lowerCamelCase		:      Any											=					via
				__lowerCamelCase						,					__lowerCamelCase		:      str											=					target
				__lowerCamelCase		:      str											=					degrees(atana(ya - ya							,						xa - xa    )    )
				via_angle %= 360
				__lowerCamelCase		:      List[Any]											=					degrees(atana(ya - ya							,						xa - xa    )    )
				target_angle %= 360
				# t-
				#  \ \
				#   \ v
				#    \|
				#     s
				# via_angle is always lower than target_angle, if direction is left.
				# If they are same, it means they are on a same line of convex hull.
				if target_angle > via_angle:
								return Direction.left
				elif target_angle == via_angle:
								return Direction.straight
				else:
								return Direction.right





def 							a_       (						lowercase    )			->list[tuple[int, int]]:






				'''simple docstring'''

				if len(lowercase    ) <= 2:
								# There is no convex hull
								raise ValueError("graham_scan: argument must contain more than 3 points."    )
				if len(lowercase    ) == 3:
								return points
				# find the lowest and the most left point
				__lowerCamelCase		:      Optional[Any]											=					0
				__lowerCamelCase						,					__lowerCamelCase		:      List[Any]											=					maxsize, maxsize
				for i, point in enumerate(lowercase    ):
								__lowerCamelCase		:      Dict											=					point[0]
								__lowerCamelCase		:      Optional[int]											=					point[1]
								if y < miny:
												__lowerCamelCase		:      Dict											=					y
												__lowerCamelCase		:      int											=					x
												__lowerCamelCase		:      List[Any]											=					i
								if y == miny and x < minx:
												__lowerCamelCase		:      Optional[int]											=					x
												__lowerCamelCase		:      Any											=					i

    # remove the lowest and the most left point from points for preparing for sort
				points.pop(lowercase    )

				__lowerCamelCase		:      Optional[Any]											=					sorted(lowercase							,						key=lambda lowercase    : angle_comparer(lowercase							,						lowercase							,						lowercase    )    )
				# This insert actually costs complexity,
				# and you should instead add (minx, miny) into stack later.
				# I'm using insert just for easy understanding.
				sorted_points.insert(0							,						(minx, miny)    )

				__lowerCamelCase		:      deque[tuple[int, int]]											=					deque()
				stack.append(sorted_points[0]    )
				stack.append(sorted_points[1]    )
				stack.append(sorted_points[2]    )
				# In any ways, the first 3 points line are towards left.
				# Because we sort them the angle from minx, miny.
				__lowerCamelCase		:      str											=					Direction.left

				for i in range(3							,						len(lowercase    )    ):
								while True:
												__lowerCamelCase		:      List[Any]											=					stack[-2]
												__lowerCamelCase		:      str											=					stack[-1]
												__lowerCamelCase		:      int											=					sorted_points[i]
												__lowerCamelCase		:      List[Any]											=					check_direction(lowercase							,						lowercase							,						lowercase    )

												if next_direction == Direction.left:
																__lowerCamelCase		:      Optional[int]											=					Direction.left
																break
												if next_direction == Direction.straight:
																if current_direction == Direction.left:
																				# We keep current_direction as left.
																				# Because if the straight line keeps as straight,
																				# we want to know if this straight line is towards left.
																				break
																elif current_direction == Direction.right:
																				# If the straight line is towards right,
																				# every previous points on those straigh line is not convex hull.
																				stack.pop()
												if next_direction == Direction.right:
																stack.pop()
								stack.append(sorted_points[i]    )
				return list(lowercase    )





