





'''simple docstring'''


import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


_lowerCamelCase				:	Tuple				    =       logging.getLogger(__name__)



class 							__UpperCAmelCase  :







  '''simple docstring'''







  def __init__(self						:      Union[str, Any]      ):
   A							  =      False







  def        A     (self						:      Any      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      Any      ):
   if not self.initialized:
    A							  =      RagRetriever(
        _lowerCAmelCase      ,      question_encoder_tokenizer=_lowerCAmelCase      ,      generator_tokenizer=_lowerCAmelCase      ,      index=_lowerCAmelCase      ,      init_retrieval=_lowerCAmelCase      ,      )
    A							  =      True







  def        A     (self						:      Optional[int]      ):
   self.retriever.index.init_index()







  def        A     (self						:      Optional[int]      ):
   # delete the old self.retriever object before assigning the new index
   del self.retriever
   A							  =      False







  def        A     (self						:      List[str]      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      List[str]      ):
   A  ,    A							  =      self.retriever._main_retrieve(_lowerCAmelCase      ,      _lowerCAmelCase      )
   A							  =      self.retriever.index.get_doc_dicts(_lowerCAmelCase      )
   return doc_ids, retrieved_doc_embeds, doc_dicts






class 							__UpperCAmelCase  (       A__						):







  '''simple docstring'''







  def __init__(self						:      Tuple      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      Tuple      ,      _lowerCAmelCase						:      str=None      ):
   if index is not None and index.is_initialized() and len(_lowerCAmelCase      ) > 0:
    raise ValueError(
        """When using Ray for distributed fine-tuning, """
        """you'll need to provide the paths instead, """
        """as the dataset and the index are loaded """
        """separately. More info in examples/rag/use_own_knowledge_dataset.py """      )

   super().__init__(
       _lowerCAmelCase      ,      question_encoder_tokenizer=_lowerCAmelCase      ,      generator_tokenizer=_lowerCAmelCase      ,      index=_lowerCAmelCase      ,      init_retrieval=_lowerCAmelCase      ,      )

   A							  =      retrieval_workers
   A							  =      question_encoder_tokenizer
   A							  =      generator_tokenizer
   if len(self.retrieval_workers      ) > 0:
    ray.get(
        [
            worker.create_rag_retriever.remote(_lowerCAmelCase      ,      _lowerCAmelCase      ,      _lowerCAmelCase      ,      _lowerCAmelCase      )
            for worker in self.retrieval_workers
        ]      )







  def        A     (self						:      List[Any]      ):
   logger.info("""initializing retrieval"""      )

   if len(self.retrieval_workers      ) > 0:
    ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]      )
   else:
    # Non-distributed training. Load index into this same process.
    self.index.init_index()







  def        A     (self						:      str      ,      _lowerCAmelCase						:      Dict      ,      _lowerCAmelCase						:      Tuple      ):
   if len(self.retrieval_workers      ) > 0:
    # Select a random retrieval actor.
    A							  =      self.retrieval_workers[random.randint(0      ,      len(self.retrieval_workers      ) - 1      )]
    A  ,    A  ,    A							  =      ray.get(
        random_worker.retrieve.remote(_lowerCAmelCase      ,      _lowerCAmelCase      )      )
   else:
    A  ,    A							  =      self._main_retrieve(_lowerCAmelCase      ,      _lowerCAmelCase      )
    A							  =      self.index.get_doc_dicts(_lowerCAmelCase      )
   return retrieved_doc_embeds, doc_ids, doc_dicts







  @classmethod
  def        A     (cls						:      Union[str, Any]      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      Optional[Any]=None      ,      **_lowerCAmelCase						:      Union[str, Any]      ):
   return super(_lowerCAmelCase      ,      cls      ).get_tokenizers(_lowerCAmelCase      ,      _lowerCAmelCase      ,      **_lowerCAmelCase      )







  @classmethod
  def        A     (cls						:      List[Any]      ,      _lowerCAmelCase						:      Optional[int]      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      str=None      ,      **_lowerCAmelCase						:      List[str]      ):
   A							  =      kwargs.pop("""config"""      ,      _lowerCAmelCase      ) or RagConfig.from_pretrained(_lowerCAmelCase      ,      **_lowerCAmelCase      )
   A							  =      RagTokenizer.from_pretrained(_lowerCAmelCase      ,      config=_lowerCAmelCase      )
   A							  =      rag_tokenizer.question_encoder
   A							  =      rag_tokenizer.generator

   if indexed_dataset is not None:
    A							  =      """custom"""
    A							  =      CustomHFIndex(config.retrieval_vector_size      ,      _lowerCAmelCase      )
   else:
    A							  =      cls._build_index(_lowerCAmelCase      )

   return cls(
       _lowerCAmelCase      ,      question_encoder_tokenizer=_lowerCAmelCase      ,      generator_tokenizer=_lowerCAmelCase      ,      retrieval_workers=_lowerCAmelCase      ,      index=_lowerCAmelCase      ,      )







  def        A     (self						:      List[Any]      ):
   logger.info("""re-loading the new dataset with embeddings"""      )
   # access from the training loop

   ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]      )

   # build the index object again
   A							  =      self._build_index(self.config      )

   ray.get(
       [
           worker.create_rag_retriever.remote(
               self.config      ,      self.question_encoder_tokenizer      ,      self.generator_tokenizer      ,      _lowerCAmelCase      )
           for worker in self.retrieval_workers
       ]      )








'''simple docstring'''


from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase				:	int				    =       logging.get_logger(__name__)



class 							__UpperCAmelCase  (       A__						):







		'''simple docstring'''







		__lowerCAmelCase 				=    ['''input_ids''', '''attention_mask''']







		def __init__(self						:      Tuple      ,      _lowerCAmelCase						:      str="[PAD]"      ,      _lowerCAmelCase						:      Dict="[BOS]"      ,      _lowerCAmelCase						:      Any="[EOS]"      ,      _lowerCAmelCase						:      List[Any]="[MASK]"      ,      _lowerCAmelCase						:      int="[CLS]"      ,      _lowerCAmelCase						:      Optional[Any]="[SEP]"      ,      _lowerCAmelCase						:      List[str]=2048      ,      **_lowerCAmelCase						:      Dict      ,      ):
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else pad_token
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else bos_token
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else eos_token
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else mask_token
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else cls_token
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else sep_token

			super().__init__(
			    pad_token=_lowerCAmelCase      ,      bos_token=_lowerCAmelCase      ,      eos_token=_lowerCAmelCase      ,      mask_token=_lowerCAmelCase      ,      cls_token=_lowerCAmelCase      ,      sep_token=_lowerCAmelCase      ,      model_max_length=_lowerCAmelCase      ,      **_lowerCAmelCase      ,      )

			A							  =      2**8  # utf is 8 bits

			# define special tokens dict
			A							  =      {
			    self.pad_token: 0,
			    self.bos_token: 1,
			    self.eos_token: 2,
			    self.mask_token: 3,
			    self.cls_token: 4,
			    self.sep_token: 5,
			}
			A							  =      len(self.special_tokens_encoder      )
			A							  =      {v: k for k, v in self.special_tokens_encoder.items()}







		def        A     (self						:      Tuple      ):
			A							  =      self.special_tokens_encoder.copy()
			vocab.update(self.added_tokens_encoder      )
			for i in range(self._utf_vocab_size      ):
				A							  =      chr(_lowerCAmelCase      )
				A							  =      i + len(self.special_tokens_encoder      )
			return vocab







		@property
		def        A     (self						:      Tuple      ):
			return self._utf_vocab_size + self._num_special_tokens







		def        A     (self						:      Union[str, Any]      ,      _lowerCAmelCase						:      List[int]      ,      _lowerCAmelCase						:      Optional[List[int]] = None      ,      _lowerCAmelCase						:      bool = False      ):
			if already_has_special_tokens:
				return super().get_special_tokens_mask(
				    token_ids_a=_lowerCAmelCase      ,      token_ids_a=_lowerCAmelCase      ,      already_has_special_tokens=_lowerCAmelCase      )

			# normal case: some special tokens
			if token_ids_a is None:
				return [1] + [0] * len(_lowerCAmelCase      ) + [1]
			return [1] + ([0] * len(_lowerCAmelCase      )) + [1] + ([0] * len(_lowerCAmelCase      )) + [1]







		def        A     (self						:      List[Any]      ,      _lowerCAmelCase						:      List[int]      ,      _lowerCAmelCase						:      Optional[List[int]] = None      ):
			if token_ids_a is None:
				return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
			else:
				return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]







		def        A     (self						:      str      ,      _lowerCAmelCase						:      str      ):
			A							  =      [chr(_lowerCAmelCase      ) for i in text.encode("""utf-8"""      )]
			return tokens







		def        A     (self						:      int      ,      _lowerCAmelCase						:      List[str]      ):
			if token in self.special_tokens_encoder:
				A							  =      self.special_tokens_encoder[token]
			elif token in self.added_tokens_encoder:
				A							  =      self.added_tokens_encoder[token]
			elif len(_lowerCAmelCase      ) != 1:
				A							  =      self.unk_token_id
			else:
				A							  =      ord(_lowerCAmelCase      ) + self._num_special_tokens
			return token_id







		def        A     (self						:      Optional[int]      ,      _lowerCAmelCase						:      Any      ):
			if index in self.special_tokens_decoder:
				A							  =      self.special_tokens_decoder[index]
			elif index in self.added_tokens_decoder:
				A							  =      self.added_tokens_decoder[index]
			else:
				A							  =      chr(index - self._num_special_tokens      )
			return token







		def        A     (self						:      Dict      ,      _lowerCAmelCase						:      List[str]      ):
			A							  =      b""""""
			for token in tokens:
				if token in self.special_tokens_decoder:
					A							  =      self.special_tokens_decoder[token].encode("""utf-8"""      )
				elif token in self.added_tokens_decoder:
					A							  =      self.special_tokens_decoder[token].encode("""utf-8"""      )
				elif token in self.special_tokens_encoder:
					A							  =      token.encode("""utf-8"""      )
				elif token in self.added_tokens_encoder:
					A							  =      token.encode("""utf-8"""      )
				else:
					A							  =      bytes([ord(_lowerCAmelCase      )]      )
				bstring += tok_string
			A							  =      bstring.decode("""utf-8"""      ,      errors="""replace"""      )
			return string







		def        A     (self						:      Tuple      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      Optional[str] = None      ):
			return ()

