


import types
from typing import List, Optional, Tuple, Union

import torch
from transformers import CLIPTextModelWithProjection, CLIPTokenizer
from transformers.models.clip.modeling_clip import CLIPTextModelOutput

from diffusers.models import PriorTransformer
from diffusers.pipelines import DiffusionPipeline, StableDiffusionImageVariationPipeline
from diffusers.schedulers import UnCLIPScheduler
from diffusers.utils import logging, randn_tensor


__snake_case             =	logging.get_logger(__name__)  # pylint: disable=invalid-name






def 		_A		(		self				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase   )				->   Optional[Any]:


     """simple docstring"""


     __UpperCamelCase   = image.to(device=_lowercase   )
     __UpperCamelCase   = image  # take image as image_embeddings
     __UpperCamelCase   = image_embeddings.unsqueeze(1   )

     # duplicate image embeddings for each generation per prompt, using mps friendly method
     __UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase   = image_embeddings.shape
     __UpperCamelCase   = image_embeddings.repeat(1				,     _lowercase				,     1   )
     __UpperCamelCase   = image_embeddings.view(bs_embed * num_images_per_prompt				,     _lowercase				,     -1   )

     if do_classifier_free_guidance:
          __UpperCamelCase   = torch.zeros_like(_lowercase   )

          # For classifier free guidance, we need to do two forward passes.
          # Here we concatenate the unconditional and text embeddings into a single batch
          # to avoid doing two forward passes
          __UpperCamelCase   = torch.cat([uncond_embeddings, image_embeddings]   )

     return image_embeddings



class   __lowerCamelCase (_a	):







       def __init__( self:   Any,A_:   PriorTransformer,A_:   CLIPTokenizer,A_:   CLIPTextModelWithProjection,A_:   UnCLIPScheduler,A_:   Optional[dict] = None,):






            '''simple docstring'''




            super().__init__()

            __UpperCamelCase   = {'image_encoder': None} if decoder_pipe_kwargs is None else decoder_pipe_kwargs

            __UpperCamelCase   = decoder_pipe_kwargs.get('torch_dtype',A_				) or prior.dtype

            __UpperCamelCase   = StableDiffusionImageVariationPipeline.from_pretrained(
                'lambdalabs/sd-image-variations-diffusers',**A_				)

            # replace `_encode_image` method
            __UpperCamelCase   = types.MethodType(_encode_image,self.decoder_pipe				)

            self.register_modules(
                prior=A_,tokenizer=A_,text_encoder=A_,prior_scheduler=A_,)







       def 		snake_case_  ( self:   Tuple,A_:   Dict,A_:   List[Any],A_:   List[str],A_:   List[Any],A_:   Optional[Union[CLIPTextModelOutput, Tuple]] = None,A_:   Optional[torch.Tensor] = None,):






            '''simple docstring'''




            if text_model_output is None:
                 __UpperCamelCase   = len(A_				) if isinstance(A_,A_				) else 1
                 # get prompt text embeddings
                 __UpperCamelCase   = self.tokenizer(
                     A_,padding='max_length',max_length=self.tokenizer.model_max_length,return_tensors='pt',)
                 __UpperCamelCase   = text_inputs.input_ids
                 __UpperCamelCase   = text_inputs.attention_mask.bool().to(A_				)

                 if text_input_ids.shape[-1] > self.tokenizer.model_max_length:
                      __UpperCamelCase   = self.tokenizer.batch_decode(text_input_ids[:, self.tokenizer.model_max_length :]				)
                      logger.warning(
                          'The following part of your input was truncated because CLIP can only handle sequences up to'
                          F''' {self.tokenizer.model_max_length} tokens: {removed_text}'''				)
                      __UpperCamelCase   = text_input_ids[:, : self.tokenizer.model_max_length]

                 __UpperCamelCase   = self.text_encoder(text_input_ids.to(A_				)				)

                 __UpperCamelCase   = text_encoder_output.text_embeds
                 __UpperCamelCase   = text_encoder_output.last_hidden_state

            else:
                 __UpperCamelCase   = text_model_output[0].shape[0]
                 __UpperCamelCase,						__UpperCamelCase   = text_model_output[0], text_model_output[1]
                 __UpperCamelCase   = text_attention_mask

            __UpperCamelCase   = text_embeddings.repeat_interleave(A_,dim=0				)
            __UpperCamelCase   = text_encoder_hidden_states.repeat_interleave(A_,dim=0				)
            __UpperCamelCase   = text_mask.repeat_interleave(A_,dim=0				)

            if do_classifier_free_guidance:
                 __UpperCamelCase   = [''] * batch_size

                 __UpperCamelCase   = self.tokenizer(
                     A_,padding='max_length',max_length=self.tokenizer.model_max_length,truncation=A_,return_tensors='pt',)
                 __UpperCamelCase   = uncond_input.attention_mask.bool().to(A_				)
                 __UpperCamelCase   = self.text_encoder(uncond_input.input_ids.to(A_				)				)

                 __UpperCamelCase   = uncond_embeddings_text_encoder_output.text_embeds
                 __UpperCamelCase   = uncond_embeddings_text_encoder_output.last_hidden_state

                 # duplicate unconditional embeddings for each generation per prompt, using mps friendly method

                 __UpperCamelCase   = uncond_embeddings.shape[1]
                 __UpperCamelCase   = uncond_embeddings.repeat(1,A_				)
                 __UpperCamelCase   = uncond_embeddings.view(batch_size * num_images_per_prompt,A_				)

                 __UpperCamelCase   = uncond_text_encoder_hidden_states.shape[1]
                 __UpperCamelCase   = uncond_text_encoder_hidden_states.repeat(1,A_,1				)
                 __UpperCamelCase   = uncond_text_encoder_hidden_states.view(
                     batch_size * num_images_per_prompt,A_,-1				)
                 __UpperCamelCase   = uncond_text_mask.repeat_interleave(A_,dim=0				)

                 # done duplicates

                 # For classifier free guidance, we need to do two forward passes.
                 # Here we concatenate the unconditional and text embeddings into a single batch
                 # to avoid doing two forward passes
                 __UpperCamelCase   = torch.cat([uncond_embeddings, text_embeddings]				)
                 __UpperCamelCase   = torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]				)

                 __UpperCamelCase   = torch.cat([uncond_text_mask, text_mask]				)

            return text_embeddings, text_encoder_hidden_states, text_mask







       @property
       def 		snake_case_  ( self:   int				):






            '''simple docstring'''




            if self.device != torch.device('meta'				) or not hasattr(self.prior,'_hf_hook'				):
                 return self.device
            for module in self.prior.modules():
                 if (
                     hasattr(A_,'_hf_hook'				)
                     and hasattr(module._hf_hook,'execution_device'				)
                     and module._hf_hook.execution_device is not None
                 ):
                      return torch.device(module._hf_hook.execution_device				)
            return self.device







       def 		snake_case_  ( self:   List[str],A_:   Optional[int],A_:   Dict,A_:   Any,A_:   Dict,A_:   Any,A_:   Optional[Any]				):






            '''simple docstring'''




            if latents is None:
                 __UpperCamelCase   = randn_tensor(A_,generator=A_,device=A_,dtype=A_				)
            else:
                 if latents.shape != shape:
                      raise ValueError(F'''Unexpected latents shape, got {latents.shape}, expected {shape}'''				)
                 __UpperCamelCase   = latents.to(A_				)

            __UpperCamelCase   = latents * scheduler.init_noise_sigma
            return latents







       def 		snake_case_  ( self:   List[Any],A_:   Optional[Union[str, torch.device]] = None				):






            '''simple docstring'''




            self.decoder_pipe.to(A_				)
            super().to(A_				)







       @torch.no_grad()
       def __call__( self:   Optional[int],A_:   Optional[Union[str, List[str]]] = None,A_:   Optional[int] = None,A_:   Optional[int] = None,A_:   int = 1,A_:   int = 25,A_:   Optional[torch.Generator] = None,A_:   Optional[torch.FloatTensor] = None,A_:   Optional[Union[CLIPTextModelOutput, Tuple]] = None,A_:   Optional[torch.Tensor] = None,A_:   float = 4.0,A_:   float = 8.0,A_:   int = 50,A_:   Optional[int] = 1,A_:   float = 0.0,A_:   Optional[str] = "pil",A_:   bool = True,):






            '''simple docstring'''




            if prompt is not None:
                 if isinstance(A_,A_				):
                      __UpperCamelCase   = 1
                 elif isinstance(A_,A_				):
                      __UpperCamelCase   = len(A_				)
                 else:
                      raise ValueError(F'''`prompt` has to be of type `str` or `list` but is {type(A_				)}'''				)
            else:
                 __UpperCamelCase   = text_model_output[0].shape[0]

            __UpperCamelCase   = self._execution_device

            __UpperCamelCase   = batch_size * num_images_per_prompt

            __UpperCamelCase   = prior_guidance_scale > 1.0 or decoder_guidance_scale > 1.0

            __UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase   = self._encode_prompt(
                A_,A_,A_,A_,A_,A_				)

            # prior

            self.prior_scheduler.set_timesteps(A_,device=A_				)
            __UpperCamelCase   = self.prior_scheduler.timesteps

            __UpperCamelCase   = self.prior.config.embedding_dim

            __UpperCamelCase   = self.prepare_latents(
                (batch_size, embedding_dim),text_embeddings.dtype,A_,A_,A_,self.prior_scheduler,)

            for i, t in enumerate(self.progress_bar(A_				)				):
                 # expand the latents if we are doing classifier free guidance
                 __UpperCamelCase   = torch.cat([prior_latents] * 2				) if do_classifier_free_guidance else prior_latents

                 __UpperCamelCase   = self.prior(
                     A_,timestep=A_,proj_embedding=A_,encoder_hidden_states=A_,attention_mask=A_,).predicted_image_embedding

                 if do_classifier_free_guidance:
                      __UpperCamelCase,						__UpperCamelCase   = predicted_image_embedding.chunk(2				)
                      __UpperCamelCase   = predicted_image_embedding_uncond + prior_guidance_scale * (
                          predicted_image_embedding_text - predicted_image_embedding_uncond
                      )

                 if i + 1 == prior_timesteps_tensor.shape[0]:
                      __UpperCamelCase   = None
                 else:
                      __UpperCamelCase   = prior_timesteps_tensor[i + 1]

                 __UpperCamelCase   = self.prior_scheduler.step(
                     A_,timestep=A_,sample=A_,generator=A_,prev_timestep=A_,).prev_sample

            __UpperCamelCase   = self.prior.post_process_latents(A_				)

            __UpperCamelCase   = prior_latents

            __UpperCamelCase   = self.decoder_pipe(
                image=A_,height=A_,width=A_,num_inference_steps=A_,guidance_scale=A_,generator=A_,output_type=A_,return_dict=A_,num_images_per_prompt=A_,eta=A_,)
            return output






import unittest

import requests

from transformers import MODEL_FOR_VISION_2_SEQ_MAPPING, TF_MODEL_FOR_VISION_2_SEQ_MAPPING, is_vision_available
from transformers.pipelines import pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_torch_available():
    from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
    __snake_case             =	False


if is_vision_available():
    from PIL import Image
else:



    class   __lowerCamelCase :







           @staticmethod
           def 		snake_case_  ( *A_:   List[str],**A_:   Any				):






                '''simple docstring'''




                pass



@is_pipeline_test
@require_vision
class   __lowerCamelCase (unittest.TestCase	):
       _lowercase       							=    MODEL_FOR_VISION_2_SEQ_MAPPING
       _lowercase       							=    TF_MODEL_FOR_VISION_2_SEQ_MAPPING







       def 		snake_case_  ( self:   Dict,A_:   int,A_:   List[str],A_:   List[str]				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model=A_,tokenizer=A_,image_processor=A_				)
            __UpperCamelCase   = [
                Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'				),
                './tests/fixtures/tests_samples/COCO/000000039769.png',
            ]
            return pipe, examples







       def 		snake_case_  ( self:   Union[str, Any],A_:   Optional[Any],A_:   Optional[int]				):






            '''simple docstring'''




            __UpperCamelCase   = pipe(A_				)
            self.assertEqual(
                A_,[
                    [{'generated_text': ANY(A_				)}],
                    [{'generated_text': ANY(A_				)}],
                ],)







       @require_tf
       def 		snake_case_  ( self:   str				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model='hf-internal-testing/tiny-random-vit-gpt2',framework='tf'				)
            __UpperCamelCase   = './tests/fixtures/tests_samples/COCO/000000039769.png'

            __UpperCamelCase   = pipe(A_				)
            self.assertEqual(
                A_,[
                    {
                        'generated_text': 'growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'
                    },
                ],)

            __UpperCamelCase   = pipe([image, image]				)
            self.assertEqual(
                A_,[
                    [
                        {
                            'generated_text': 'growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'
                        }
                    ],
                    [
                        {
                            'generated_text': 'growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'
                        }
                    ],
                ],)

            __UpperCamelCase   = pipe(A_,max_new_tokens=1				)
            self.assertEqual(
                A_,[{'generated_text': 'growth'}],)







       @require_torch
       def 		snake_case_  ( self:   Any				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model='hf-internal-testing/tiny-random-vit-gpt2'				)
            __UpperCamelCase   = './tests/fixtures/tests_samples/COCO/000000039769.png'

            __UpperCamelCase   = pipe(A_				)
            self.assertEqual(
                A_,[
                    {
                        'generated_text': 'growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'
                    },
                ],)

            __UpperCamelCase   = pipe([image, image]				)
            self.assertEqual(
                A_,[
                    [
                        {
                            'generated_text': 'growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'
                        }
                    ],
                    [
                        {
                            'generated_text': 'growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'
                        }
                    ],
                ],)







       @require_torch
       def 		snake_case_  ( self:   str				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model='hf-internal-testing/tiny-random-BlipForConditionalGeneration'				)
            __UpperCamelCase   = './tests/fixtures/tests_samples/COCO/000000039769.png'
            __UpperCamelCase   = 'a photo of'

            __UpperCamelCase   = pipe(A_,prompt=A_				)
            self.assertTrue(outputs[0]['generated_text'].startswith(A_				)				)







       @slow
       @require_torch
       def 		snake_case_  ( self:   Dict				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model='ydshieh/vit-gpt2-coco-en'				)
            __UpperCamelCase   = './tests/fixtures/tests_samples/COCO/000000039769.png'

            __UpperCamelCase   = pipe(A_				)
            self.assertEqual(A_,[{'generated_text': 'a cat laying on a blanket next to a cat laying on a bed '}]				)

            __UpperCamelCase   = pipe([image, image]				)
            self.assertEqual(
                A_,[
                    [{'generated_text': 'a cat laying on a blanket next to a cat laying on a bed '}],
                    [{'generated_text': 'a cat laying on a blanket next to a cat laying on a bed '}],
                ],)







       @slow
       @require_torch
       def 		snake_case_  ( self:   Dict				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model='Salesforce/blip-image-captioning-base'				)
            __UpperCamelCase   = 'https://huggingface.co/datasets/sayakpaul/sample-datasets/resolve/main/pokemon.png'
            __UpperCamelCase   = Image.open(requests.get(A_,stream=A_				).raw				)

            __UpperCamelCase   = pipe(A_				)
            self.assertEqual(A_,[{'generated_text': 'a pink pokemon pokemon with a blue shirt and a blue shirt'}]				)







       @slow
       @require_torch
       def 		snake_case_  ( self:   Dict				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model='microsoft/git-base-coco'				)
            __UpperCamelCase   = 'https://huggingface.co/datasets/sayakpaul/sample-datasets/resolve/main/pokemon.png'
            __UpperCamelCase   = Image.open(requests.get(A_,stream=A_				).raw				)

            __UpperCamelCase   = pipe(A_				)
            self.assertEqual(A_,[{'generated_text': 'a cartoon of a purple character.'}]				)







       @slow
       @require_torch
       def 		snake_case_  ( self:   Dict				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model='Salesforce/blip-image-captioning-base'				)
            __UpperCamelCase   = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg'
            __UpperCamelCase   = Image.open(requests.get(A_,stream=A_				).raw				)

            __UpperCamelCase   = 'a photography of'

            __UpperCamelCase   = pipe(A_,prompt=A_				)
            self.assertEqual(A_,[{'generated_text': 'a photography of a volcano'}]				)

            with self.assertRaises(A_				):
                 __UpperCamelCase   = pipe([image, image],prompt=[prompt, prompt]				)







       @slow
       @require_torch
       def 		snake_case_  ( self:   List[Any]				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model='microsoft/git-base-coco'				)
            __UpperCamelCase   = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg'
            __UpperCamelCase   = Image.open(requests.get(A_,stream=A_				).raw				)

            __UpperCamelCase   = 'a photo of a'

            __UpperCamelCase   = pipe(A_,prompt=A_				)
            self.assertEqual(A_,[{'generated_text': 'a photo of a tent with a tent and a tent in the background.'}]				)

            with self.assertRaises(A_				):
                 __UpperCamelCase   = pipe([image, image],prompt=[prompt, prompt]				)







       @unittest.skipIf(
           not is_torch_greater_or_equal_than_1_11,reason='`Pix2StructImageProcessor` requires `torch>=1.11.0`.'				)
       @slow
       @require_torch
       def 		snake_case_  ( self:   List[Any]				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model='google/pix2struct-ai2d-base'				)
            __UpperCamelCase   = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg'
            __UpperCamelCase   = Image.open(requests.get(A_,stream=A_				).raw				)

            __UpperCamelCase   = 'What does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud'

            __UpperCamelCase   = pipe(A_,prompt=A_				)
            self.assertEqual(A_,[{'generated_text': 'ash cloud'}]				)

            with self.assertRaises(A_				):
                 __UpperCamelCase   = pipe([image, image],prompt=[prompt, prompt]				)







       @slow
       @require_tf
       def 		snake_case_  ( self:   Any				):






            '''simple docstring'''




            __UpperCamelCase   = pipeline('image-to-text',model='ydshieh/vit-gpt2-coco-en',framework='tf'				)
            __UpperCamelCase   = './tests/fixtures/tests_samples/COCO/000000039769.png'

            __UpperCamelCase   = pipe(A_				)
            self.assertEqual(A_,[{'generated_text': 'a cat laying on a blanket next to a cat laying on a bed '}]				)

            __UpperCamelCase   = pipe([image, image]				)
            self.assertEqual(
                A_,[
                    [{'generated_text': 'a cat laying on a blanket next to a cat laying on a bed '}],
                    [{'generated_text': 'a cat laying on a blanket next to a cat laying on a bed '}],
                ],)


