




"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A 		=			logging.get_logger(__name__)

__A 		=			{
    "google/reformer-crime-and-punishment": (
        "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/config.json"
    ),
    "google/reformer-enwik8": "https://huggingface.co/google/reformer-enwik8/resolve/main/config.json",
}
class 			lowerCamelCase__    (  lowerCamelCase_			):
			a__ :      Dict								=     """reformer"""
			a__ :      Tuple								=     ["""past_buckets_states"""]
			a__ :      Optional[int]								=     {}


			def __init__(     self				, SCREAMING_SNAKE_CASE=64				, SCREAMING_SNAKE_CASE=["local", "lsh", "local", "lsh", "local", "lsh"]				, SCREAMING_SNAKE_CASE=1.0				, SCREAMING_SNAKE_CASE=True				, SCREAMING_SNAKE_CASE=[64, 64]				, SCREAMING_SNAKE_CASE=[64, 192]				, SCREAMING_SNAKE_CASE=0				, SCREAMING_SNAKE_CASE=2				, SCREAMING_SNAKE_CASE=512				, SCREAMING_SNAKE_CASE=None				, SCREAMING_SNAKE_CASE="relu"				, SCREAMING_SNAKE_CASE=0.05				, SCREAMING_SNAKE_CASE=256				, SCREAMING_SNAKE_CASE=0.02				, SCREAMING_SNAKE_CASE=False				, SCREAMING_SNAKE_CASE=1E-12				, SCREAMING_SNAKE_CASE=1				, SCREAMING_SNAKE_CASE=0				, SCREAMING_SNAKE_CASE=0.05				, SCREAMING_SNAKE_CASE=64				, SCREAMING_SNAKE_CASE=64				, SCREAMING_SNAKE_CASE=0.0				, SCREAMING_SNAKE_CASE=1				, SCREAMING_SNAKE_CASE=0				, SCREAMING_SNAKE_CASE=4_096				, SCREAMING_SNAKE_CASE=12				, SCREAMING_SNAKE_CASE=None				, SCREAMING_SNAKE_CASE=1				, SCREAMING_SNAKE_CASE=0				, SCREAMING_SNAKE_CASE=320				, SCREAMING_SNAKE_CASE=False				, SCREAMING_SNAKE_CASE=True				, SCREAMING_SNAKE_CASE=None				, **SCREAMING_SNAKE_CASE				, ):



					"""simple docstring"""







					snake_case		:						List[str]								=				hash_seed
					snake_case		:						Union[str, Any]								=				vocab_size
					snake_case		:						Optional[Any]								=				attention_head_size
					snake_case		:						Optional[int]								=				hidden_size
					snake_case		:						Union[str, Any]								=				num_attention_heads
					snake_case		:						List[str]								=				num_hashes
					snake_case		:						str								=				len(SCREAMING_SNAKE_CASE							)
					snake_case		:						str								=				tuple(SCREAMING_SNAKE_CASE							) if isinstance(SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE							) else num_buckets
					snake_case		:						int								=				lsh_attn_chunk_length
					snake_case		:						List[str]								=				local_attn_chunk_length
					snake_case		:						Any								=				lsh_num_chunks_after
					snake_case		:						Union[str, Any]								=				lsh_num_chunks_before
					snake_case		:						Tuple								=				local_num_chunks_after
					snake_case		:						Optional[int]								=				local_num_chunks_before
					snake_case		:						Tuple								=				hidden_act
					snake_case		:						List[str]								=				feed_forward_size
					snake_case		:						Tuple								=				hidden_dropout_prob
					snake_case		:						Optional[Any]								=				lsh_attention_probs_dropout_prob
					snake_case		:						Any								=				local_attention_probs_dropout_prob
					snake_case		:						List[Any]								=				max_position_embeddings
					snake_case		:						Tuple								=				initializer_range
					snake_case		:						Union[str, Any]								=				layer_norm_eps
					snake_case		:						str								=				axial_pos_embds
					snake_case		:						List[str]								=				tuple(SCREAMING_SNAKE_CASE							)
					snake_case		:						List[Any]								=				tuple(SCREAMING_SNAKE_CASE							)
					snake_case		:						int								=				axial_norm_std
					snake_case		:						List[Any]								=				chunk_size_lm_head
					snake_case		:						Optional[Any]								=				attn_layers
					snake_case		:						int								=				use_cache
					snake_case		:						Any								=				classifier_dropout
					super().__init__(
					    pad_token_id=SCREAMING_SNAKE_CASE				, eos_token_id=SCREAMING_SNAKE_CASE				, is_decoder=SCREAMING_SNAKE_CASE				, tie_word_embeddings=SCREAMING_SNAKE_CASE				, **SCREAMING_SNAKE_CASE				, )







"""simple docstring"""

import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A 		=			logging.get_logger(__name__)
__A 		=			{"vocab_file": "spiece.model"}

__A 		=			{
    "vocab_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/spiece.model",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/spiece.model",
    },
}

__A 		=			{
    "google/fnet-base": 512,
    "google/fnet-large": 512,
}

__A 		=			"▁"
class 			lowerCamelCase__    (  lowerCamelCase_			):
			a__ :      Dict								=     VOCAB_FILES_NAMES
			a__ :      Tuple								=     PRETRAINED_VOCAB_FILES_MAP
			a__ :      int								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			a__ :      List[Any]								=     ["""input_ids""", """token_type_ids"""]


			def __init__(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE=False				, SCREAMING_SNAKE_CASE=True				, SCREAMING_SNAKE_CASE=True				, SCREAMING_SNAKE_CASE="<unk>"				, SCREAMING_SNAKE_CASE="[SEP]"				, SCREAMING_SNAKE_CASE="<pad>"				, SCREAMING_SNAKE_CASE="[CLS]"				, SCREAMING_SNAKE_CASE="[MASK]"				, SCREAMING_SNAKE_CASE = None				, **SCREAMING_SNAKE_CASE				, ):



					"""simple docstring"""







					snake_case		:						Dict								=				(
					    AddedToken(SCREAMING_SNAKE_CASE				, lstrip=SCREAMING_SNAKE_CASE				, rstrip=SCREAMING_SNAKE_CASE				, normalized=SCREAMING_SNAKE_CASE							)
					    if isinstance(SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE							)
					    else mask_token
					)

					snake_case		:						Optional[int]								=				{} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    do_lower_case=SCREAMING_SNAKE_CASE				, remove_space=SCREAMING_SNAKE_CASE				, keep_accents=SCREAMING_SNAKE_CASE				, unk_token=SCREAMING_SNAKE_CASE				, sep_token=SCREAMING_SNAKE_CASE				, pad_token=SCREAMING_SNAKE_CASE				, cls_token=SCREAMING_SNAKE_CASE				, mask_token=SCREAMING_SNAKE_CASE				, sp_model_kwargs=self.sp_model_kwargs				, **SCREAMING_SNAKE_CASE				, )

					snake_case		:						List[str]								=				do_lower_case
					snake_case		:						Any								=				remove_space
					snake_case		:						Union[str, Any]								=				keep_accents
					snake_case		:						Optional[int]								=				vocab_file

					snake_case		:						Dict								=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
					self.sp_model.Load(SCREAMING_SNAKE_CASE							)


			@property
			def 							lowerCamelCase_				(     self							):



					"""simple docstring"""







					return len(self.sp_model							)


			def 							lowerCamelCase_				(     self							):



					"""simple docstring"""







					snake_case		:						Optional[int]								=				{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE							): i for i in range(self.vocab_size							)}
					vocab.update(self.added_tokens_encoder							)
					return vocab


			def __getstate__(     self							):



					"""simple docstring"""







					snake_case		:						Tuple								=				self.__dict__.copy()
					snake_case		:						List[Any]								=				None
					return state


			def __setstate__(     self				, SCREAMING_SNAKE_CASE							):



					"""simple docstring"""







					snake_case		:						Union[str, Any]								=				d

					# for backward compatibility
					if not hasattr(self				, "sp_model_kwargs"							):
							snake_case		:						List[Any]								=				{}

					snake_case		:						Tuple								=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
					self.sp_model.Load(self.vocab_file							)


			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE							):



					"""simple docstring"""







					if self.remove_space:
							snake_case		:						int								=				" ".join(inputs.strip().split()							)
					else:
							snake_case		:						Union[str, Any]								=				inputs
					snake_case		:						Union[str, Any]								=				outputs.replace("``"				, "\""							).replace("''"				, "\""							)

					if not self.keep_accents:
							snake_case		:						List[Any]								=				unicodedata.normalize("NFKD"				, SCREAMING_SNAKE_CASE							)
							snake_case		:						int								=				"".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE							)]							)
					if self.do_lower_case:
							snake_case		:						Union[str, Any]								=				outputs.lower()

					return outputs


			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE							):



					"""simple docstring"""







					snake_case		:						List[str]								=				self.preprocess_text(SCREAMING_SNAKE_CASE							)
					snake_case		:						List[str]								=				self.sp_model.encode(SCREAMING_SNAKE_CASE				, out_type=SCREAMING_SNAKE_CASE							)
					snake_case		:						Optional[Any]								=				[]
					for piece in pieces:
							if len(SCREAMING_SNAKE_CASE							) > 1 and piece[-1] == str(","							) and piece[-2].isdigit():
									snake_case		:						Union[str, Any]								=				self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE				, ""							)							)
									if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
											if len(cur_pieces[0]							) == 1:
													snake_case		:						List[Any]								=				cur_pieces[1:]
											else:
													snake_case		:						Dict								=				cur_pieces[0][1:]
									cur_pieces.append(piece[-1]							)
									new_pieces.extend(SCREAMING_SNAKE_CASE							)
							else:
									new_pieces.append(SCREAMING_SNAKE_CASE							)

					return new_pieces


			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE							):



					"""simple docstring"""







					return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE							)


			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE							):



					"""simple docstring"""







					return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE							)


			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE							):



					"""simple docstring"""







					snake_case		:						Dict								=				[]
					snake_case		:						Any								=				""
					snake_case		:						Dict								=				False
					for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
									if not prev_is_special:
											out_string += " "
									out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE							) + token
									snake_case		:						Optional[int]								=				True
									snake_case		:						Union[str, Any]								=				[]
							else:
									current_sub_tokens.append(SCREAMING_SNAKE_CASE							)
									snake_case		:						Tuple								=				False
					out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE							)
					return out_string.strip()


			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE = False				, SCREAMING_SNAKE_CASE = None				, SCREAMING_SNAKE_CASE = True				, **SCREAMING_SNAKE_CASE				, ):



					"""simple docstring"""







					snake_case		:						int								=				kwargs.pop("use_source_tokenizer"				, SCREAMING_SNAKE_CASE							)

					snake_case		:						Tuple								=				self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE				, skip_special_tokens=SCREAMING_SNAKE_CASE							)

					# To avoid mixing byte-level and unicode for byte-level BPT
					# we need to build string separately for added tokens and byte-level tokens
					# cf. https://github.com/huggingface/transformers/issues/1133
					snake_case		:						Any								=				[]
					snake_case		:						Union[str, Any]								=				[]
					for token in filtered_tokens:
							if skip_special_tokens and token in self.all_special_ids:
									continue
							if token in self.added_tokens_encoder:
									if current_sub_text:
											sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE							)							)
											snake_case		:						Optional[Any]								=				[]
									sub_texts.append(SCREAMING_SNAKE_CASE							)
							else:
									current_sub_text.append(SCREAMING_SNAKE_CASE							)
					if current_sub_text:
							sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE							)							)

					# Mimic the behavior of the Rust tokenizer:
					# No space after <unk>
					if spaces_between_special_tokens:
							snake_case		:						Any								=				re.sub(r"(<unk>) "				, r"\1"				, " ".join(SCREAMING_SNAKE_CASE							)							)
					else:
							snake_case		:						Optional[int]								=				"".join(SCREAMING_SNAKE_CASE							)

					snake_case		:						Optional[int]								=				(
					    clean_up_tokenization_spaces
					    if clean_up_tokenization_spaces is not None
					    else self.clean_up_tokenization_spaces
					)
					if clean_up_tokenization_spaces:
							snake_case		:						List[Any]								=				self.clean_up_tokenization(SCREAMING_SNAKE_CASE							)
							return clean_text
					else:
							return text


			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE = None							):



					"""simple docstring"""







					snake_case		:						Tuple								=				[self.sep_token_id]
					snake_case		:						List[str]								=				[self.cls_token_id]
					if token_ids_a is None:
							return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep


			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE = None				, SCREAMING_SNAKE_CASE = False							):



					"""simple docstring"""







					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=SCREAMING_SNAKE_CASE				, token_ids_a=SCREAMING_SNAKE_CASE				, already_has_special_tokens=SCREAMING_SNAKE_CASE							)

					if token_ids_a is not None:
							return [1] + ([0] * len(SCREAMING_SNAKE_CASE							)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE							)) + [1]
					return [1] + ([0] * len(SCREAMING_SNAKE_CASE							)) + [1]


			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE = None							):



					"""simple docstring"""







					snake_case		:						Dict								=				[self.sep_token_id]
					snake_case		:						Optional[int]								=				[self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep							) * [0]
					return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]



			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE = None							):



					"""simple docstring"""







					if not os.path.isdir(SCREAMING_SNAKE_CASE							):
							logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
							return
					snake_case		:						Tuple								=				os.path.join(
					    SCREAMING_SNAKE_CASE				, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

					if os.path.abspath(self.vocab_file							) != os.path.abspath(SCREAMING_SNAKE_CASE							) and os.path.isfile(self.vocab_file							):
							copyfile(self.vocab_file				, SCREAMING_SNAKE_CASE							)
					elif not os.path.isfile(self.vocab_file							):
							with open(SCREAMING_SNAKE_CASE				, "wb"							) as fi:
									snake_case		:						List[str]								=				self.sp_model.serialized_model_proto()
									fi.write(SCREAMING_SNAKE_CASE							)

					return (out_vocab_file,)

