'''simple docstring'''

import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase_  		=			logging.get_logger(__name__)
lowercase_  		=			{'''vocab_file''': '''spiece.model'''}

lowercase_  		=			{
    '''vocab_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
    },
}

lowercase_  		=			{
    '''google/fnet-base''': 512,
    '''google/fnet-large''': 512,
}

lowercase_  		=			'''▁'''
class 	SCREAMING_SNAKE_CASE      (    lowerCamelCase__):







		"""simple docstring"""
		lowercase :   Any							=				VOCAB_FILES_NAMES
		lowercase :   Dict							=				PRETRAINED_VOCAB_FILES_MAP
		lowercase :   Union[str, Any]							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowercase :   int							=				['input_ids', 'token_type_ids']






		def __init__( self		,    __A		,    __A=False		,    __A=True		,    __A=True		,    __A="<unk>"		,    __A="[SEP]"		,    __A="<pad>"		,    __A="[CLS]"		,    __A="[MASK]"		,    __A = None		,    **__A		,    )			->  List[Any]:
								_lowerCAmelCase					    =(
								    AddedToken(_snake_case		,    lstrip=_snake_case		,    rstrip=_snake_case		,    normalized=_snake_case					)
								    if isinstance(_snake_case		,    _snake_case					)
								    else mask_token
								)

								_lowerCAmelCase					    ={} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    do_lower_case=_snake_case		,    remove_space=_snake_case		,    keep_accents=_snake_case		,    unk_token=_snake_case		,    sep_token=_snake_case		,    pad_token=_snake_case		,    cls_token=_snake_case		,    mask_token=_snake_case		,    sp_model_kwargs=self.sp_model_kwargs		,    **_snake_case		,    )

								_lowerCAmelCase					    =do_lower_case
								_lowerCAmelCase					    =remove_space
								_lowerCAmelCase					    =keep_accents
								_lowerCAmelCase					    =vocab_file

								_lowerCAmelCase					    =spm.SentencePieceProcessor(**self.sp_model_kwargs					)
								self.sp_model.Load(_snake_case					)






		@property
		def 		UpperCamelCase__							( self					)			->  int:
								return len(self.sp_model					)






		def 		UpperCamelCase__							( self					)			->  int:
								_lowerCAmelCase					    ={self.convert_ids_to_tokens(_snake_case					): i for i in range(self.vocab_size					)}
								vocab.update(self.added_tokens_encoder					)
								return vocab






		def __getstate__( self					)			->  str:
								_lowerCAmelCase					    =self.__dict__.copy()
								_lowerCAmelCase					    =None
								return state






		def __setstate__( self		,    __A					)			->  Optional[int]:
								_lowerCAmelCase					    =d

								# for backward compatibility
								if not hasattr(self		,    'sp_model_kwargs'					):
														_lowerCAmelCase					    ={}

								_lowerCAmelCase					    =spm.SentencePieceProcessor(**self.sp_model_kwargs					)
								self.sp_model.Load(self.vocab_file					)






		def 		UpperCamelCase__							( self		,    __A					)			->  List[Any]:





								if self.remove_space:
														_lowerCAmelCase					    =' '.join(inputs.strip().split()					)
								else:
														_lowerCAmelCase					    =inputs
								_lowerCAmelCase					    =outputs.replace('``'		,    '\"'					).replace('\'\''		,    '\"'					)

								if not self.keep_accents:
														_lowerCAmelCase					    =unicodedata.normalize('NFKD'		,    _snake_case					)
														_lowerCAmelCase					    =''.join([c for c in outputs if not unicodedata.combining(_snake_case					)]					)
								if self.do_lower_case:
														_lowerCAmelCase					    =outputs.lower()

								return outputs






		def 		UpperCamelCase__							( self		,    __A					)			->  str:
								_lowerCAmelCase					    =self.preprocess_text(_snake_case					)
								_lowerCAmelCase					    =self.sp_model.encode(_snake_case		,    out_type=_snake_case					)
								_lowerCAmelCase					    =[]
								for piece in pieces:
														if len(_snake_case					) > 1 and piece[-1] == str(','					) and piece[-2].isdigit():
																				_lowerCAmelCase					    =self.sp_model.EncodeAsPieces(piece[:-1].replace(_snake_case		,    ''					)					)
																				if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																										if len(cur_pieces[0]					) == 1:
																																_lowerCAmelCase					    =cur_pieces[1:]
																										else:
																																_lowerCAmelCase					    =cur_pieces[0][1:]
																				cur_pieces.append(piece[-1]					)
																				new_pieces.extend(_snake_case					)
														else:
																				new_pieces.append(_snake_case					)

								return new_pieces






		def 		UpperCamelCase__							( self		,    __A					)			->  Optional[int]:
								return self.sp_model.PieceToId(_snake_case					)






		def 		UpperCamelCase__							( self		,    __A					)			->  Optional[Any]:
								return self.sp_model.IdToPiece(_snake_case					)






		def 		UpperCamelCase__							( self		,    __A					)			->  Union[str, Any]:
								_lowerCAmelCase					    =[]
								_lowerCAmelCase					    =''
								_lowerCAmelCase					    =False
								for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																				if not prev_is_special:
																										out_string += " "
																				out_string += self.sp_model.decode(_snake_case					) + token
																				_lowerCAmelCase					    =True
																				_lowerCAmelCase					    =[]
														else:
																				current_sub_tokens.append(_snake_case					)
																				_lowerCAmelCase					    =False
								out_string += self.sp_model.decode(_snake_case					)
								return out_string.strip()






		def 		UpperCamelCase__							( self		,    __A		,    __A = False		,    __A = None		,    __A = True		,    **__A		,    )			->  Optional[int]:
								_lowerCAmelCase					    =kwargs.pop('use_source_tokenizer'		,    _snake_case					)

								_lowerCAmelCase					    =self.convert_ids_to_tokens(_snake_case		,    skip_special_tokens=_snake_case					)

								# To avoid mixing byte-level and unicode for byte-level BPT
								# we need to build string separately for added tokens and byte-level tokens
								# cf. https://github.com/huggingface/transformers/issues/1133
								_lowerCAmelCase					    =[]
								_lowerCAmelCase					    =[]
								for token in filtered_tokens:
														if skip_special_tokens and token in self.all_special_ids:
																				continue
														if token in self.added_tokens_encoder:
																				if current_sub_text:
																										sub_texts.append(self.convert_tokens_to_string(_snake_case					)					)
																										_lowerCAmelCase					    =[]
																				sub_texts.append(_snake_case					)
														else:
																				current_sub_text.append(_snake_case					)
								if current_sub_text:
														sub_texts.append(self.convert_tokens_to_string(_snake_case					)					)

								# Mimic the behavior of the Rust tokenizer:
								# No space after <unk>
								if spaces_between_special_tokens:
														_lowerCAmelCase					    =re.sub(r'(<unk>) '		,    r'\1'		,    ' '.join(_snake_case					)					)
								else:
														_lowerCAmelCase					    =''.join(_snake_case					)

								_lowerCAmelCase					    =(
								    clean_up_tokenization_spaces
								    if clean_up_tokenization_spaces is not None
								    else self.clean_up_tokenization_spaces
								)
								if clean_up_tokenization_spaces:
														_lowerCAmelCase					    =self.clean_up_tokenization(_snake_case					)
														return clean_text
								else:
														return text






		def 		UpperCamelCase__							( self		,    __A		,    __A = None					)			->  Optional[Any]:
								_lowerCAmelCase					    =[self.sep_token_id]
								_lowerCAmelCase					    =[self.cls_token_id]
								if token_ids_a is None:
														return cls + token_ids_a + sep
								return cls + token_ids_a + sep + token_ids_a + sep






		def 		UpperCamelCase__							( self		,    __A		,    __A = None		,    __A = False					)			->  Optional[Any]:





								if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=_snake_case		,    token_ids_a=_snake_case		,    already_has_special_tokens=_snake_case					)

								if token_ids_a is not None:
														return [1] + ([0] * len(_snake_case					)) + [1] + ([0] * len(_snake_case					)) + [1]
								return [1] + ([0] * len(_snake_case					)) + [1]






		def 		UpperCamelCase__							( self		,    __A		,    __A = None					)			->  Dict:
								_lowerCAmelCase					    =[self.sep_token_id]
								_lowerCAmelCase					    =[self.cls_token_id]

								if token_ids_a is None:
														return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






		def 		UpperCamelCase__							( self		,    __A		,    __A = None					)			->  Optional[int]:





								if not os.path.isdir(_snake_case					):
														logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
														return
								_lowerCAmelCase					    =os.path.join(
								    _snake_case		,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

								if os.path.abspath(self.vocab_file					) != os.path.abspath(_snake_case					) and os.path.isfile(self.vocab_file					):
														copyfile(self.vocab_file		,    _snake_case					)
								elif not os.path.isfile(self.vocab_file					):
														with open(_snake_case		,    'wb'					) as fi:
																				_lowerCAmelCase					    =self.sp_model.serialized_model_proto()
																				fi.write(_snake_case					)

								return (out_vocab_file,)










import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


A__       =			"""block_records.npy"""


A__       =			logging.get_logger(__name__)


def 						_UpperCAmelCase							( snake_case      ,		snake_case     ):





					"""simple docstring"""

					import tensorflow.compat.va as tf

					_lowerCAmelCase         =       tf.data.TFRecordDataset(snake_case      ,		buffer_size=5_12 * 10_24 * 10_24     )
					_lowerCAmelCase         =       blocks_dataset.batch(snake_case      ,		drop_remainder=snake_case     )
					_lowerCAmelCase         =       next(blocks_dataset.take(1     ).as_numpy_iterator()     )

					return np_record





class    __lowerCAmelCase					:




				def __init__(				self		,							_snake_case		,							_snake_case		,							_snake_case=2		,							_snake_case=1000		,							_snake_case=100		,							_snake_case=100000		,							):



									"""simple docstring"""





									from scann.scann_ops.py.scann_ops_pybind import builder as Builder

									_lowerCAmelCase         =       Builder(db=_snake_case		,							num_neighbors=_snake_case		,							distance_measure="""dot_product"""						)
									_lowerCAmelCase         =       builder.tree(
									    num_leaves=_snake_case		,							num_leaves_to_search=_snake_case		,							training_sample_size=_snake_case						)
									_lowerCAmelCase         =       builder.score_ah(dimensions_per_block=_snake_case						)

									_lowerCAmelCase         =       builder.build()




				def 				snake_case  (				self		,							_snake_case						):



									"""simple docstring"""





									_lowerCAmelCase      ,      _lowerCAmelCase         =       self.searcher.search_batched(question_projection.detach().cpu()						)
									return retrieved_block_ids.astype("""int64"""						)





class    __lowerCAmelCase					:




				def __init__(				self		,							_snake_case		,							_snake_case						):



									"""simple docstring"""





									super().__init__()
									_lowerCAmelCase         =       block_records
									_lowerCAmelCase         =       tokenizer




				def __call__(				self		,							_snake_case		,							_snake_case		,							_snake_case		,							_snake_case=None		,							_snake_case="pt"						):



									"""simple docstring"""





									_lowerCAmelCase         =       np.take(self.block_records		,							indices=_snake_case		,							axis=0						)

									_lowerCAmelCase         =       self.tokenizer.decode(question_input_ids[0]		,							skip_special_tokens=_snake_case						)

									_lowerCAmelCase         =       []
									_lowerCAmelCase         =       []
									for retrieved_block in retrieved_blocks:
														text.append(_snake_case						)
														text_pair.append(retrieved_block.decode()						)

									_lowerCAmelCase         =       self.tokenizer(
									    _snake_case		,							_snake_case		,							padding=_snake_case		,							truncation=_snake_case		,							return_special_tokens_mask=_snake_case		,							max_length=_snake_case						)
									_lowerCAmelCase         =       concat_inputs.convert_to_tensors(_snake_case						)

									if answer_ids is not None:
														return self.block_has_answer(_snake_case		,							_snake_case						) + (concat_inputs_tensors,)
									else:
														return (None, None, None, concat_inputs_tensors)




				@classmethod
				def 				snake_case  (				cls		,							_snake_case		,							*_snake_case		,							**_snake_case						):



									"""simple docstring"""





									if os.path.isdir(_snake_case						):
														_lowerCAmelCase         =       os.path.join(_snake_case		,							_REALM_BLOCK_RECORDS_FILENAME						)
									else:
														_lowerCAmelCase         =       hf_hub_download(
														    repo_id=_snake_case		,							filename=_REALM_BLOCK_RECORDS_FILENAME		,							**_snake_case						)
									_lowerCAmelCase         =       np.load(_snake_case		,							allow_pickle=_snake_case						)

									_lowerCAmelCase         =       AutoTokenizer.from_pretrained(_snake_case		,							*_snake_case		,							**_snake_case						)

									return cls(_snake_case		,							_snake_case						)




				def 				snake_case  (				self		,							_snake_case						):



									"""simple docstring"""





									np.save(os.path.join(_snake_case		,							_REALM_BLOCK_RECORDS_FILENAME						)		,							self.block_records						)
									# save tokenizer
									self.tokenizer.save_pretrained(_snake_case						)




				def 				snake_case  (				self		,							_snake_case		,							_snake_case						):



									"""simple docstring"""





									_lowerCAmelCase         =       []
									_lowerCAmelCase         =       []
									_lowerCAmelCase         =       []
									_lowerCAmelCase         =       0

									for input_id in concat_inputs.input_ids:
														_lowerCAmelCase         =       input_id.tolist()
														# Check answers between two [SEP] tokens
														_lowerCAmelCase         =       input_id_list.index(self.tokenizer.sep_token_id						)
														_lowerCAmelCase         =       first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id						)

														start_pos.append([]						)
														end_pos.append([]						)
														for answer in answer_ids:
																			for idx in range(first_sep_idx + 1		,							_snake_case						):
																								if answer[0] == input_id_list[idx]:
																													if input_id_list[idx : idx + len(_snake_case						)] == answer:
																																		start_pos[-1].append(_snake_case						)
																																		end_pos[-1].append(idx + len(_snake_case						) - 1						)

														if len(start_pos[-1]						) == 0:
																			has_answers.append(_snake_case						)
														else:
																			has_answers.append(_snake_case						)
																			if len(start_pos[-1]						) > max_answers:
																								_lowerCAmelCase         =       len(start_pos[-1]						)

        # Pad -1 to max_answers
									for start_pos_, end_pos_ in zip(_snake_case		,							_snake_case						):
														if len(start_pos_						) < max_answers:
																			_lowerCAmelCase         =       [-1] * (max_answers - len(start_pos_						))
																			start_pos_ += padded
																			end_pos_ += padded
									return has_answers, start_pos, end_pos



