






'''simple docstring'''

import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


__snake_case       :   Tuple  		=    logging.getLogger(__name__)
class 							lowercase_    :







    def __init__(	self				)				->       int:





     """simple docstring"""

     UpperCAmelCase_				       =					False







    def   lowerCamelCase_    (	self       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__				)				->       Any:





     """simple docstring"""

     if not self.initialized:
      UpperCAmelCase_				       =					RagRetriever(
          _lowerCAmelCase       ,     question_encoder_tokenizer=_lowerCAmelCase       ,     generator_tokenizer=_lowerCAmelCase       ,     index=_lowerCAmelCase       ,     init_retrieval=_lowerCAmelCase       ,     )
      UpperCAmelCase_				       =					True







    def   lowerCamelCase_    (	self				)				->       Dict:





     """simple docstring"""

     self.retriever.index.init_index()







    def   lowerCamelCase_    (	self				)				->       str:





     """simple docstring"""

     del self.retriever
     UpperCAmelCase_				       =					False







    def   lowerCamelCase_    (	self       ,     UpperCamelCase__       ,     UpperCamelCase__				)				->       Union[str, Any]:





     """simple docstring"""

     UpperCAmelCase_       ,       UpperCAmelCase_				       =					self.retriever._main_retrieve(_lowerCAmelCase       ,     _lowerCAmelCase				)
     UpperCAmelCase_				       =					self.retriever.index.get_doc_dicts(_lowerCAmelCase				)
     return doc_ids, retrieved_doc_embeds, doc_dicts


class 							lowercase_    (     A__						):







    def __init__(	self       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__=None				)				->       Optional[Any]:





     """simple docstring"""

     if index is not None and index.is_initialized() and len(_lowerCAmelCase				) > 0:
      raise ValueError(
          "When using Ray for distributed fine-tuning, "
          "you'll need to provide the paths instead, "
          "as the dataset and the index are loaded "
          "separately. More info in examples/rag/use_own_knowledge_dataset.py "				)

     super().__init__(
         _lowerCAmelCase       ,     question_encoder_tokenizer=_lowerCAmelCase       ,     generator_tokenizer=_lowerCAmelCase       ,     index=_lowerCAmelCase       ,     init_retrieval=_lowerCAmelCase       ,     )

     UpperCAmelCase_				       =					retrieval_workers
     UpperCAmelCase_				       =					question_encoder_tokenizer
     UpperCAmelCase_				       =					generator_tokenizer
     if len(self.retrieval_workers				) > 0:
      ray.get(
          [
              worker.create_rag_retriever.remote(_lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase				)
              for worker in self.retrieval_workers
          ]				)







    def   lowerCamelCase_    (	self				)				->       Union[str, Any]:





     """simple docstring"""

     logger.info("initializing retrieval"				)

     if len(self.retrieval_workers				) > 0:
      ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]				)
     else:
      # Non-distributed training. Load index into this same process.
      self.index.init_index()







    def   lowerCamelCase_    (	self       ,     UpperCamelCase__       ,     UpperCamelCase__				)				->       List[Any]:





     """simple docstring"""

     if len(self.retrieval_workers				) > 0:
      # Select a random retrieval actor.
      UpperCAmelCase_				       =					self.retrieval_workers[random.randint(0       ,     len(self.retrieval_workers				) - 1				)]
      UpperCAmelCase_       ,       UpperCAmelCase_       ,       UpperCAmelCase_				       =					ray.get(
          random_worker.retrieve.remote(_lowerCAmelCase       ,     _lowerCAmelCase				)				)
     else:
      UpperCAmelCase_       ,       UpperCAmelCase_				       =					self._main_retrieve(_lowerCAmelCase       ,     _lowerCAmelCase				)
      UpperCAmelCase_				       =					self.index.get_doc_dicts(_lowerCAmelCase				)
     return retrieved_doc_embeds, doc_ids, doc_dicts







    @classmethod
    def   lowerCamelCase_    (	cls       ,     UpperCamelCase__       ,     UpperCamelCase__=None       ,     **UpperCamelCase__				)				->       Optional[Any]:





     """simple docstring"""

     return super(_lowerCAmelCase       ,     cls				).get_tokenizers(_lowerCAmelCase       ,     _lowerCAmelCase       ,     **_lowerCAmelCase				)







    @classmethod
    def   lowerCamelCase_    (	cls       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__=None       ,     **UpperCamelCase__				)				->       Optional[Any]:





     """simple docstring"""

     UpperCAmelCase_				       =					kwargs.pop("config"       ,     _lowerCAmelCase				) or RagConfig.from_pretrained(_lowerCAmelCase       ,     **_lowerCAmelCase				)
     UpperCAmelCase_				       =					RagTokenizer.from_pretrained(_lowerCAmelCase       ,     config=_lowerCAmelCase				)
     UpperCAmelCase_				       =					rag_tokenizer.question_encoder
     UpperCAmelCase_				       =					rag_tokenizer.generator

     if indexed_dataset is not None:
      UpperCAmelCase_				       =					"custom"
      UpperCAmelCase_				       =					CustomHFIndex(config.retrieval_vector_size       ,     _lowerCAmelCase				)
     else:
      UpperCAmelCase_				       =					cls._build_index(_lowerCAmelCase				)

     return cls(
         _lowerCAmelCase       ,     question_encoder_tokenizer=_lowerCAmelCase       ,     generator_tokenizer=_lowerCAmelCase       ,     retrieval_workers=_lowerCAmelCase       ,     index=_lowerCAmelCase       ,     )







    def   lowerCamelCase_    (	self				)				->       Tuple:





     """simple docstring"""

     logger.info("re-loading the new dataset with embeddings"				)
     # access from the training loop

     ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]				)

     # build the index object again
     UpperCAmelCase_				       =					self._build_index(self.config				)

     ray.get(
         [
             worker.create_rag_retriever.remote(
                 self.config       ,     self.question_encoder_tokenizer       ,     self.generator_tokenizer       ,     _lowerCAmelCase				)
             for worker in self.retrieval_workers
         ]				)

'''simple docstring'''

from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


__snake_case       :   Any  		=    logging.get_logger(__name__)


if is_vision_available():
      import PIL
class 							lowercase_    (     _A						):
    a_             =   ["""pixel_values"""]







    def __init__(	self       ,     UpperCamelCase__ = True       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = PILImageResampling.BICUBIC       ,     UpperCamelCase__ = True       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = True       ,     UpperCamelCase__ = 1 / 2_5_5       ,     UpperCamelCase__ = True       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = True       ,     **UpperCamelCase__       ,     )				->       None:





     """simple docstring"""

     super().__init__(**UpperCamelCase__				)
     UpperCAmelCase_				       =					size if size is not None else {"shortest_edge": 2_2_4}
     UpperCAmelCase_				       =					get_size_dict(UpperCamelCase__       ,     default_to_square=UpperCamelCase__				)
     UpperCAmelCase_				       =					crop_size if crop_size is not None else {"height": 2_2_4, "width": 2_2_4}
     UpperCAmelCase_				       =					get_size_dict(UpperCamelCase__				)

     UpperCAmelCase_				       =					do_resize
     UpperCAmelCase_				       =					size
     UpperCAmelCase_				       =					resample
     UpperCAmelCase_				       =					do_center_crop
     UpperCAmelCase_				       =					crop_size
     UpperCAmelCase_				       =					do_rescale
     UpperCAmelCase_				       =					rescale_factor
     UpperCAmelCase_				       =					do_normalize
     UpperCAmelCase_				       =					image_mean if image_mean is not None else OPENAI_CLIP_MEAN
     UpperCAmelCase_				       =					image_std if image_std is not None else OPENAI_CLIP_STD
     UpperCAmelCase_				       =					do_convert_rgb







    def   lowerCamelCase_    (	self       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__ = PILImageResampling.BICUBIC       ,     UpperCamelCase__ = None       ,     **UpperCamelCase__       ,     )				->       np.ndarray:





     """simple docstring"""

     UpperCAmelCase_				       =					get_size_dict(UpperCamelCase__       ,     default_to_square=UpperCamelCase__				)
     UpperCAmelCase_				       =					get_resize_output_image_size(
         UpperCamelCase__       ,     size=(size["height"], size["width"])       ,     default_to_square=UpperCamelCase__				)
     return resize(UpperCamelCase__       ,     size=UpperCamelCase__       ,     resample=UpperCamelCase__       ,     data_format=UpperCamelCase__       ,     **UpperCamelCase__				)







    def   lowerCamelCase_    (	self       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__ = None       ,     **UpperCamelCase__       ,     )				->       np.ndarray:





     """simple docstring"""

     UpperCAmelCase_				       =					get_size_dict(UpperCamelCase__				)
     return center_crop(UpperCamelCase__       ,     size=(size["height"], size["width"])       ,     data_format=UpperCamelCase__       ,     **UpperCamelCase__				)







    def   lowerCamelCase_    (	self       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__ = None       ,     **UpperCamelCase__       ,     )				->       int:





     """simple docstring"""

     return rescale(UpperCamelCase__       ,     scale=UpperCamelCase__       ,     data_format=UpperCamelCase__       ,     **UpperCamelCase__				)







    def   lowerCamelCase_    (	self       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__       ,     UpperCamelCase__ = None       ,     **UpperCamelCase__       ,     )				->       np.ndarray:





     """simple docstring"""

     return normalize(UpperCamelCase__       ,     mean=UpperCamelCase__       ,     std=UpperCamelCase__       ,     data_format=UpperCamelCase__       ,     **UpperCamelCase__				)







    def   lowerCamelCase_    (	self       ,     UpperCamelCase__       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = None       ,     UpperCamelCase__ = ChannelDimension.FIRST       ,     **UpperCamelCase__       ,     )				->       PIL.Image.Image:





     """simple docstring"""

     UpperCAmelCase_				       =					do_resize if do_resize is not None else self.do_resize
     UpperCAmelCase_				       =					size if size is not None else self.size
     UpperCAmelCase_				       =					get_size_dict(UpperCamelCase__       ,     default_to_square=UpperCamelCase__				)
     UpperCAmelCase_				       =					resample if resample is not None else self.resample
     UpperCAmelCase_				       =					do_center_crop if do_center_crop is not None else self.do_center_crop
     UpperCAmelCase_				       =					crop_size if crop_size is not None else self.crop_size
     UpperCAmelCase_				       =					get_size_dict(UpperCamelCase__				)
     UpperCAmelCase_				       =					do_rescale if do_rescale is not None else self.do_rescale
     UpperCAmelCase_				       =					rescale_factor if rescale_factor is not None else self.rescale_factor
     UpperCAmelCase_				       =					do_normalize if do_normalize is not None else self.do_normalize
     UpperCAmelCase_				       =					image_mean if image_mean is not None else self.image_mean
     UpperCAmelCase_				       =					image_std if image_std is not None else self.image_std
     UpperCAmelCase_				       =					do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

     UpperCAmelCase_				       =					make_list_of_images(UpperCamelCase__				)

     if not valid_images(UpperCamelCase__				):
      raise ValueError(
          "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
          "torch.Tensor, tf.Tensor or jax.ndarray."				)

     if do_resize and size is None:
      raise ValueError("Size must be specified if do_resize is True."				)

     if do_center_crop and crop_size is None:
      raise ValueError("Crop size must be specified if do_center_crop is True."				)

     if do_rescale and rescale_factor is None:
      raise ValueError("Rescale factor must be specified if do_rescale is True."				)

     if do_normalize and (image_mean is None or image_std is None):
      raise ValueError("Image mean and std must be specified if do_normalize is True."				)

     # PIL RGBA images are converted to RGB
     if do_convert_rgb:
      UpperCAmelCase_				       =					[convert_to_rgb(UpperCamelCase__				) for image in images]

     # All transformations expect numpy arrays.
     UpperCAmelCase_				       =					[to_numpy_array(UpperCamelCase__				) for image in images]

     if do_resize:
      UpperCAmelCase_				       =					[self.resize(image=UpperCamelCase__       ,     size=UpperCamelCase__       ,     resample=UpperCamelCase__				) for image in images]

     if do_center_crop:
      UpperCAmelCase_				       =					[self.center_crop(image=UpperCamelCase__       ,     size=UpperCamelCase__				) for image in images]

     if do_rescale:
      UpperCAmelCase_				       =					[self.rescale(image=UpperCamelCase__       ,     scale=UpperCamelCase__				) for image in images]

     if do_normalize:
      UpperCAmelCase_				       =					[self.normalize(image=UpperCamelCase__       ,     mean=UpperCamelCase__       ,     std=UpperCamelCase__				) for image in images]

     UpperCAmelCase_				       =					[to_channel_dimension_format(UpperCamelCase__       ,     UpperCamelCase__				) for image in images]

     UpperCAmelCase_				       =					{"pixel_values": images}
     return BatchFeature(data=UpperCamelCase__       ,     tensor_type=UpperCamelCase__				)
