






import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


lowerCamelCase_		  =						"""block_records.npy"""


lowerCamelCase_		  =						logging.get_logger(__name__)
def   lowerCamelCase      (		a_				,		a_			)			->    np.ndarray:
							import tensorflow.compat.va as tf

							lowerCAmelCase_       		=  tf.data.TFRecordDataset(a_				,		buffer_size=512 * 1_024 * 1_024			)
							lowerCAmelCase_       		=  blocks_dataset.batch(a_				,		drop_remainder=a_			)
							lowerCAmelCase_       		=  next(blocks_dataset.take(1			).as_numpy_iterator()			)

							return np_record




class 							a_				:


							'''simple docstring'''






							def __init__(	self    ,    lowercase_    ,    lowercase_    ,    lowercase_=2    ,    lowercase_=1_0_0_0    ,    lowercase_=1_0_0    ,    lowercase_=1_0_0_0_0_0    ,    )      ->      List[str]:
														'''simple docstring'''

														from scann.scann_ops.py.scann_ops_pybind import builder as Builder

														lowerCAmelCase_       		=  Builder(db=lowercase_    ,    num_neighbors=lowercase_    ,    distance_measure='dot_product'     )
														lowerCAmelCase_       		=  builder.tree(
														    num_leaves=lowercase_    ,    num_leaves_to_search=lowercase_    ,    training_sample_size=lowercase_     )
														lowerCAmelCase_       		=  builder.score_ah(dimensions_per_block=lowercase_     )

														lowerCAmelCase_       		=  builder.build()






							def   _lowercase (	self    ,    lowercase_     )      ->      int:
														'''simple docstring'''

														lowerCAmelCase_		,     lowerCAmelCase_       		=  self.searcher.search_batched(question_projection.detach().cpu()     )
														return retrieved_block_ids.astype('int64'     )




class 							a_				:


							'''simple docstring'''






							def __init__(	self    ,    lowercase_    ,    lowercase_     )      ->      List[str]:
														'''simple docstring'''

														super().__init__()
														lowerCAmelCase_       		=  block_records
														lowerCAmelCase_       		=  tokenizer

							def __call__(	self    ,    lowercase_    ,    lowercase_    ,    lowercase_    ,    lowercase_=None    ,    lowercase_="pt"     )      ->      Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  np.take(self.block_records    ,    indices=lowercase_    ,    axis=0     )

														lowerCAmelCase_       		=  self.tokenizer.decode(question_input_ids[0]    ,    skip_special_tokens=lowercase_     )

														lowerCAmelCase_       		=  []
														lowerCAmelCase_       		=  []
														for retrieved_block in retrieved_blocks:
																					text.append(lowercase_     )
																					text_pair.append(retrieved_block.decode()     )

														lowerCAmelCase_       		=  self.tokenizer(
														    lowercase_    ,    lowercase_    ,    padding=lowercase_    ,    truncation=lowercase_    ,    return_special_tokens_mask=lowercase_    ,    max_length=lowercase_     )
														lowerCAmelCase_       		=  concat_inputs.convert_to_tensors(lowercase_     )

														if answer_ids is not None:
																					return self.block_has_answer(lowercase_    ,    lowercase_     ) + (concat_inputs_tensors,)
														else:
																					return (None, None, None, concat_inputs_tensors)

							@classmethod
							def   _lowercase (	cls    ,    lowercase_    ,    *lowercase_    ,    **lowercase_     )      ->      List[str]:
														'''simple docstring'''

														if os.path.isdir(lowercase_     ):
																					lowerCAmelCase_       		=  os.path.join(lowercase_    ,    _REALM_BLOCK_RECORDS_FILENAME     )
														else:
																					lowerCAmelCase_       		=  hf_hub_download(
																					    repo_id=lowercase_    ,    filename=_REALM_BLOCK_RECORDS_FILENAME    ,    **lowercase_     )
														lowerCAmelCase_       		=  np.load(lowercase_    ,    allow_pickle=lowercase_     )

														lowerCAmelCase_       		=  AutoTokenizer.from_pretrained(lowercase_    ,    *lowercase_    ,    **lowercase_     )

														return cls(lowercase_    ,    lowercase_     )

							def   _lowercase (	self    ,    lowercase_     )      ->      List[Any]:
														'''simple docstring'''

														np.save(os.path.join(lowercase_    ,    _REALM_BLOCK_RECORDS_FILENAME     )    ,    self.block_records     )
														# save tokenizer
														self.tokenizer.save_pretrained(lowercase_     )






							def   _lowercase (	self    ,    lowercase_    ,    lowercase_     )      ->      Tuple:
														'''simple docstring'''

														lowerCAmelCase_       		=  []
														lowerCAmelCase_       		=  []
														lowerCAmelCase_       		=  []
														lowerCAmelCase_       		=  0

														for input_id in concat_inputs.input_ids:
																					lowerCAmelCase_       		=  input_id.tolist()
																					# Check answers between two [SEP] tokens
																					lowerCAmelCase_       		=  input_id_list.index(self.tokenizer.sep_token_id     )
																					lowerCAmelCase_       		=  first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id     )

																					start_pos.append([]     )
																					end_pos.append([]     )
																					for answer in answer_ids:
																												for idx in range(first_sep_idx + 1    ,    lowercase_     ):
																																			if answer[0] == input_id_list[idx]:
																																										if input_id_list[idx : idx + len(lowercase_     )] == answer:
																																																	start_pos[-1].append(lowercase_     )
																																																	end_pos[-1].append(idx + len(lowercase_     ) - 1     )

																					if len(start_pos[-1]     ) == 0:
																												has_answers.append(lowercase_     )
																					else:
																												has_answers.append(lowercase_     )
																												if len(start_pos[-1]     ) > max_answers:
																																			lowerCAmelCase_       		=  len(start_pos[-1]     )

        # Pad -1 to max_answers
														for start_pos_, end_pos_ in zip(lowercase_    ,    lowercase_     ):
																					if len(start_pos_     ) < max_answers:
																												lowerCAmelCase_       		=  [-1] * (max_answers - len(start_pos_     ))
																												start_pos_ += padded
																												end_pos_ += padded
														return has_answers, start_pos, end_pos














import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class 							a_				(		a_	):


							'''simple docstring'''






							__a:			int									=					['''image_processor''', '''tokenizer''']
							__a:			int									=					'''VideoMAEImageProcessor'''
							__a:			Dict									=					('''CLIPTokenizer''', '''CLIPTokenizerFast''')

							def __init__(	self    ,    lowercase_=None    ,    lowercase_=None    ,    **lowercase_     )      ->      Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  None
														if "feature_extractor" in kwargs:
																					warnings.warn(
																					    'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
																					    ' instead.'    ,    lowercase_    ,    )
																					lowerCAmelCase_       		=  kwargs.pop('feature_extractor'     )

														lowerCAmelCase_       		=  image_processor if image_processor is not None else feature_extractor
														if image_processor is None:
																					raise ValueError('You need to specify an `image_processor`.'     )
														if tokenizer is None:
																					raise ValueError('You need to specify a `tokenizer`.'     )

														super().__init__(lowercase_    ,    lowercase_     )
														lowerCAmelCase_       		=  self.image_processor

							def __call__(	self    ,    lowercase_=None    ,    lowercase_=None    ,    lowercase_=None    ,    **lowercase_     )      ->      Optional[int]:
														'''simple docstring'''

														if text is None and videos is None:
																					raise ValueError('You have to specify either text or videos. Both cannot be none.'     )

														if text is not None:
																					lowerCAmelCase_       		=  self.tokenizer(lowercase_    ,    return_tensors=lowercase_    ,    **lowercase_     )

														if videos is not None:
																					lowerCAmelCase_       		=  self.image_processor(lowercase_    ,    return_tensors=lowercase_    ,    **lowercase_     )

														if text is not None and videos is not None:
																					lowerCAmelCase_       		=  image_features.pixel_values
																					return encoding
														elif text is not None:
																					return encoding
														else:
																					return BatchEncoding(data=dict(**lowercase_     )    ,    tensor_type=lowercase_     )

							def   _lowercase (	self    ,    *lowercase_    ,    **lowercase_     )      ->      Union[str, Any]:
														'''simple docstring'''

														return self.tokenizer.batch_decode(*lowercase_    ,    **lowercase_     )

							def   _lowercase (	self    ,    *lowercase_    ,    **lowercase_     )      ->      str:
														'''simple docstring'''

														return self.tokenizer.decode(*lowercase_    ,    **lowercase_     )

							@property
							def   _lowercase (	self     )      ->      Any:
														'''simple docstring'''

														return ["input_ids", "attention_mask", "position_ids", "pixel_values"]

							@property
							def   _lowercase (	self     )      ->      List[str]:
														'''simple docstring'''

														warnings.warn(
														    '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'    ,    lowercase_    ,    )
														return self.image_processor_class






							@property
							def   _lowercase (	self     )      ->      int:
														'''simple docstring'''

														warnings.warn(
														    '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'    ,    lowercase_    ,    )
														return self.image_processor



