



import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


A__       =			"""block_records.npy"""


A__       =			logging.get_logger(__name__)


def 						_UpperCAmelCase							( snake_case      ,		snake_case     ):





					"""simple docstring"""

					import tensorflow.compat.va as tf

					_lowerCAmelCase         =       tf.data.TFRecordDataset(snake_case      ,		buffer_size=5_12 * 10_24 * 10_24     )
					_lowerCAmelCase         =       blocks_dataset.batch(snake_case      ,		drop_remainder=snake_case     )
					_lowerCAmelCase         =       next(blocks_dataset.take(1     ).as_numpy_iterator()     )

					return np_record





class    __lowerCAmelCase					:




				def __init__(				self		,							_snake_case		,							_snake_case		,							_snake_case=2		,							_snake_case=1000		,							_snake_case=100		,							_snake_case=100000		,							):



									"""simple docstring"""





									from scann.scann_ops.py.scann_ops_pybind import builder as Builder

									_lowerCAmelCase         =       Builder(db=_snake_case		,							num_neighbors=_snake_case		,							distance_measure="""dot_product"""						)
									_lowerCAmelCase         =       builder.tree(
									    num_leaves=_snake_case		,							num_leaves_to_search=_snake_case		,							training_sample_size=_snake_case						)
									_lowerCAmelCase         =       builder.score_ah(dimensions_per_block=_snake_case						)

									_lowerCAmelCase         =       builder.build()




				def 				snake_case  (				self		,							_snake_case						):



									"""simple docstring"""





									_lowerCAmelCase      ,      _lowerCAmelCase         =       self.searcher.search_batched(question_projection.detach().cpu()						)
									return retrieved_block_ids.astype("""int64"""						)





class    __lowerCAmelCase					:




				def __init__(				self		,							_snake_case		,							_snake_case						):



									"""simple docstring"""





									super().__init__()
									_lowerCAmelCase         =       block_records
									_lowerCAmelCase         =       tokenizer




				def __call__(				self		,							_snake_case		,							_snake_case		,							_snake_case		,							_snake_case=None		,							_snake_case="pt"						):



									"""simple docstring"""





									_lowerCAmelCase         =       np.take(self.block_records		,							indices=_snake_case		,							axis=0						)

									_lowerCAmelCase         =       self.tokenizer.decode(question_input_ids[0]		,							skip_special_tokens=_snake_case						)

									_lowerCAmelCase         =       []
									_lowerCAmelCase         =       []
									for retrieved_block in retrieved_blocks:
														text.append(_snake_case						)
														text_pair.append(retrieved_block.decode()						)

									_lowerCAmelCase         =       self.tokenizer(
									    _snake_case		,							_snake_case		,							padding=_snake_case		,							truncation=_snake_case		,							return_special_tokens_mask=_snake_case		,							max_length=_snake_case						)
									_lowerCAmelCase         =       concat_inputs.convert_to_tensors(_snake_case						)

									if answer_ids is not None:
														return self.block_has_answer(_snake_case		,							_snake_case						) + (concat_inputs_tensors,)
									else:
														return (None, None, None, concat_inputs_tensors)




				@classmethod
				def 				snake_case  (				cls		,							_snake_case		,							*_snake_case		,							**_snake_case						):



									"""simple docstring"""





									if os.path.isdir(_snake_case						):
														_lowerCAmelCase         =       os.path.join(_snake_case		,							_REALM_BLOCK_RECORDS_FILENAME						)
									else:
														_lowerCAmelCase         =       hf_hub_download(
														    repo_id=_snake_case		,							filename=_REALM_BLOCK_RECORDS_FILENAME		,							**_snake_case						)
									_lowerCAmelCase         =       np.load(_snake_case		,							allow_pickle=_snake_case						)

									_lowerCAmelCase         =       AutoTokenizer.from_pretrained(_snake_case		,							*_snake_case		,							**_snake_case						)

									return cls(_snake_case		,							_snake_case						)




				def 				snake_case  (				self		,							_snake_case						):



									"""simple docstring"""





									np.save(os.path.join(_snake_case		,							_REALM_BLOCK_RECORDS_FILENAME						)		,							self.block_records						)
									# save tokenizer
									self.tokenizer.save_pretrained(_snake_case						)




				def 				snake_case  (				self		,							_snake_case		,							_snake_case						):



									"""simple docstring"""





									_lowerCAmelCase         =       []
									_lowerCAmelCase         =       []
									_lowerCAmelCase         =       []
									_lowerCAmelCase         =       0

									for input_id in concat_inputs.input_ids:
														_lowerCAmelCase         =       input_id.tolist()
														# Check answers between two [SEP] tokens
														_lowerCAmelCase         =       input_id_list.index(self.tokenizer.sep_token_id						)
														_lowerCAmelCase         =       first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id						)

														start_pos.append([]						)
														end_pos.append([]						)
														for answer in answer_ids:
																			for idx in range(first_sep_idx + 1		,							_snake_case						):
																								if answer[0] == input_id_list[idx]:
																													if input_id_list[idx : idx + len(_snake_case						)] == answer:
																																		start_pos[-1].append(_snake_case						)
																																		end_pos[-1].append(idx + len(_snake_case						) - 1						)

														if len(start_pos[-1]						) == 0:
																			has_answers.append(_snake_case						)
														else:
																			has_answers.append(_snake_case						)
																			if len(start_pos[-1]						) > max_answers:
																								_lowerCAmelCase         =       len(start_pos[-1]						)

        # Pad -1 to max_answers
									for start_pos_, end_pos_ in zip(_snake_case		,							_snake_case						):
														if len(start_pos_						) < max_answers:
																			_lowerCAmelCase         =       [-1] * (max_answers - len(start_pos_						))
																			start_pos_ += padded
																			end_pos_ += padded
									return has_answers, start_pos, end_pos








import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
	from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

	from .test_feature_extraction_speechta import floats_list


A__       =			get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")





@require_torch
class    __lowerCAmelCase					(	unittest.TestCase			):




				def 				snake_case  (				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       tempfile.mkdtemp()

									_lowerCAmelCase         =       SpeechTaTokenizer(_snake_case						)
									tokenizer.save_pretrained(self.tmpdirname						)

									_lowerCAmelCase         =       {
									    """feature_size""": 1,
									    """padding_value""": 0.0,
									    """sampling_rate""": 16000,
									    """do_normalize""": False,
									    """num_mel_bins""": 80,
									    """hop_length""": 16,
									    """win_length""": 64,
									    """win_function""": """hann_window""",
									    """fmin""": 80,
									    """fmax""": 7600,
									    """mel_floor""": 1e-10,
									    """reduction_factor""": 2,
									    """return_attention_mask""": True,
									}

									_lowerCAmelCase         =       os.path.join(self.tmpdirname		,							_snake_case						)
									with open(self.feature_extraction_file		,							"""w"""		,							encoding="""utf-8"""						) as fp:
														fp.write(json.dumps(_snake_case						) + """\n"""						)




				def 				snake_case  (				self		,							**_snake_case						):



									"""simple docstring"""





									return SpeechTaTokenizer.from_pretrained(self.tmpdirname		,							**_snake_case						)




				def 				snake_case  (				self		,							**_snake_case						):



									"""simple docstring"""





									return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname		,							**_snake_case						)




				def 				snake_case  (				self						):



									"""simple docstring"""





									shutil.rmtree(self.tmpdirname						)




				def 				snake_case  (				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       self.get_tokenizer()
									_lowerCAmelCase         =       self.get_feature_extractor()

									_lowerCAmelCase         =       SpeechTaProcessor(tokenizer=_snake_case		,							feature_extractor=_snake_case						)

									processor.save_pretrained(self.tmpdirname						)
									_lowerCAmelCase         =       SpeechTaProcessor.from_pretrained(self.tmpdirname						)

									self.assertEqual(processor.tokenizer.get_vocab()		,							tokenizer.get_vocab()						)
									self.assertIsInstance(processor.tokenizer		,							_snake_case						)

									self.assertEqual(processor.feature_extractor.to_json_string()		,							feature_extractor.to_json_string()						)
									self.assertIsInstance(processor.feature_extractor		,							_snake_case						)




				def 				snake_case  (				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       SpeechTaProcessor(tokenizer=self.get_tokenizer()		,							feature_extractor=self.get_feature_extractor()						)
									processor.save_pretrained(self.tmpdirname						)

									_lowerCAmelCase         =       self.get_tokenizer(bos_token="""(BOS)"""		,							eos_token="""(EOS)"""						)
									_lowerCAmelCase         =       self.get_feature_extractor(do_normalize=_snake_case		,							padding_value=1.0						)

									_lowerCAmelCase         =       SpeechTaProcessor.from_pretrained(
									    self.tmpdirname		,							bos_token="""(BOS)"""		,							eos_token="""(EOS)"""		,							do_normalize=_snake_case		,							padding_value=1.0						)

									self.assertEqual(processor.tokenizer.get_vocab()		,							tokenizer_add_kwargs.get_vocab()						)
									self.assertIsInstance(processor.tokenizer		,							_snake_case						)

									self.assertEqual(processor.feature_extractor.to_json_string()		,							feature_extractor_add_kwargs.to_json_string()						)
									self.assertIsInstance(processor.feature_extractor		,							_snake_case						)




				def 				snake_case  (				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       self.get_feature_extractor()
									_lowerCAmelCase         =       self.get_tokenizer()

									_lowerCAmelCase         =       SpeechTaProcessor(tokenizer=_snake_case		,							feature_extractor=_snake_case						)

									_lowerCAmelCase         =       floats_list((3, 1000)						)

									_lowerCAmelCase         =       feature_extractor(audio=_snake_case		,							return_tensors="""np"""						)
									_lowerCAmelCase         =       processor(audio=_snake_case		,							return_tensors="""np"""						)

									for key in input_feat_extract.keys():
														self.assertAlmostEqual(input_feat_extract[key].sum()		,							input_processor[key].sum()		,							delta=1e-2						)




				def 				snake_case  (				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       self.get_feature_extractor()
									_lowerCAmelCase         =       self.get_tokenizer()

									_lowerCAmelCase         =       SpeechTaProcessor(tokenizer=_snake_case		,							feature_extractor=_snake_case						)

									_lowerCAmelCase         =       floats_list((3, 1000)						)

									_lowerCAmelCase         =       feature_extractor(audio_target=_snake_case		,							return_tensors="""np"""						)
									_lowerCAmelCase         =       processor(audio_target=_snake_case		,							return_tensors="""np"""						)

									for key in input_feat_extract.keys():
														self.assertAlmostEqual(input_feat_extract[key].sum()		,							input_processor[key].sum()		,							delta=1e-2						)




				def 				snake_case  (				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       self.get_feature_extractor()
									_lowerCAmelCase         =       self.get_tokenizer()

									_lowerCAmelCase         =       SpeechTaProcessor(tokenizer=_snake_case		,							feature_extractor=_snake_case						)

									_lowerCAmelCase         =       """This is a test string"""

									_lowerCAmelCase         =       processor(text=_snake_case						)
									_lowerCAmelCase         =       tokenizer(_snake_case						)

									for key in encoded_tok.keys():
														self.assertListEqual(encoded_tok[key]		,							encoded_processor[key]						)




				def 				snake_case  (				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       self.get_feature_extractor()
									_lowerCAmelCase         =       self.get_tokenizer()

									_lowerCAmelCase         =       SpeechTaProcessor(tokenizer=_snake_case		,							feature_extractor=_snake_case						)

									_lowerCAmelCase         =       """This is a test string"""

									_lowerCAmelCase         =       processor(text_target=_snake_case						)
									_lowerCAmelCase         =       tokenizer(_snake_case						)

									for key in encoded_tok.keys():
														self.assertListEqual(encoded_tok[key]		,							encoded_processor[key]						)




				def 				snake_case  (				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       self.get_feature_extractor()
									_lowerCAmelCase         =       self.get_tokenizer()

									_lowerCAmelCase         =       SpeechTaProcessor(tokenizer=_snake_case		,							feature_extractor=_snake_case						)

									_lowerCAmelCase         =       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

									_lowerCAmelCase         =       processor.batch_decode(_snake_case						)
									_lowerCAmelCase         =       tokenizer.batch_decode(_snake_case						)

									self.assertListEqual(_snake_case		,							_snake_case						)




				def 				snake_case  (				self						):



									"""simple docstring"""





									_lowerCAmelCase         =       self.get_feature_extractor()
									_lowerCAmelCase         =       self.get_tokenizer()

									_lowerCAmelCase         =       SpeechTaProcessor(tokenizer=_snake_case		,							feature_extractor=_snake_case						)

									self.assertListEqual(
									    processor.model_input_names		,							feature_extractor.model_input_names		,							msg="""`processor` and `feature_extractor` model input names do not match"""		,							)



