





'''simple docstring'''


from __future__ import annotations

import copy
import os
import tempfile
from importlib import import_module
from math import isnan

from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow

from ..test_modeling_tf_common import ids_tensor


if is_tf_available():
				import numpy as np
				import tensorflow as tf

				from transformers import (
				    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
				    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
				    TF_MODEL_FOR_MASKED_LM_MAPPING,
				    TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
				    TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
				    TF_MODEL_FOR_PRETRAINING_MAPPING,
				    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
				    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
				    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
				    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
				    TFSharedEmbeddings,
				)

				if _tf_gpu_memory_limit is not None:
								_lowerCamelCase				:	Union[str, Any]				    =       tf.config.list_physical_devices('GPU')
								for gpu in gpus:
												# Restrict TensorFlow to only allocate x GB of memory on the GPUs
												try:
																tf.config.set_logical_device_configuration(
																    gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)]
																)
																_lowerCamelCase				:	int				    =       tf.config.list_logical_devices('GPU')
																print('Logical GPUs', logical_gpus)
												except RuntimeError as e:
																# Virtual devices must be set before GPUs have been initialized
																print(e)



@require_tf
class 							__UpperCAmelCase  :







		'''simple docstring'''







		__lowerCAmelCase 				=    None
		__lowerCAmelCase 				=    ()
		__lowerCAmelCase 				=    ()
		__lowerCAmelCase 				=    True
		__lowerCAmelCase 				=    True
		__lowerCAmelCase 				=    True
		__lowerCAmelCase 				=    False







		def        A     (self						:      str      ,      _lowerCAmelCase						:      List[Any]      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      Tuple=False      ):
			A							  =      copy.deepcopy(_lowerCAmelCase      )

			if model_class in get_values(_lowerCAmelCase      ):
				A							  =      {
				    k: tf.tile(tf.expand_dims(_lowerCAmelCase      ,      1      )      ,      (1, self.model_tester.num_choices) + (1,) * (v.ndim - 1)      )
				    if isinstance(_lowerCAmelCase      ,      tf.Tensor      ) and v.ndim > 0
				    else v
				    for k, v in inputs_dict.items()
				}

			if return_labels:
				if model_class in get_values(_lowerCAmelCase      ):
					A							  =      tf.ones(self.model_tester.batch_size      ,      dtype=tf.intaa      )
				elif model_class in get_values(_lowerCAmelCase      ):
					A							  =      tf.zeros(self.model_tester.batch_size      ,      dtype=tf.intaa      )
					A							  =      tf.zeros(self.model_tester.batch_size      ,      dtype=tf.intaa      )
				elif model_class in [
				    *get_values(_lowerCAmelCase      ),
				    *get_values(_lowerCAmelCase      ),
				]:
					A							  =      tf.zeros(self.model_tester.batch_size      ,      dtype=tf.intaa      )
				elif model_class in get_values(_lowerCAmelCase      ):
					A							  =      tf.zeros(self.model_tester.batch_size      ,      dtype=tf.intaa      )
				elif model_class in [
				    *get_values(_lowerCAmelCase      ),
				    *get_values(_lowerCAmelCase      ),
				    *get_values(_lowerCAmelCase      ),
				    *get_values(_lowerCAmelCase      ),
				    *get_values(_lowerCAmelCase      ),
				]:
					A							  =      tf.zeros(
					    (self.model_tester.batch_size, self.model_tester.seq_length)      ,      dtype=tf.intaa      )
			return inputs_dict







		@slow
		def        A     (self						:      Optional[Any]      ):
			A  ,    A							  =      self.model_tester.prepare_config_and_inputs_for_common()
			for model_class in self.all_model_classes[:2]:
				A							  =      self._prepare_for_class(_lowerCAmelCase      ,      _lowerCAmelCase      )
				A							  =      model_class(_lowerCAmelCase      )

				@tf.function
				def run_in_graph_mode():
					return model(_lowerCAmelCase      )

				A							  =      run_in_graph_mode()
				self.assertIsNotNone(_lowerCAmelCase      )







		@slow
		def        A     (self						:      List[str]      ):
			A  ,    A							  =      self.model_tester.prepare_config_and_inputs_for_common()
			for model_class in self.all_model_classes[:2]:
				A							  =      self._prepare_for_class(_lowerCAmelCase      ,      _lowerCAmelCase      )
				A							  =      model_class(_lowerCAmelCase      )

				@tf.function(experimental_compile=_lowerCAmelCase      )
				def run_in_graph_mode():
					return model(_lowerCAmelCase      )

				A							  =      run_in_graph_mode()
				self.assertIsNotNone(_lowerCAmelCase      )







		@slow
		def        A     (self						:      Tuple      ):
			# This is a copy of the test_keras_fit method, but we use XLA compilation instead of eager
			A  ,    A							  =      self.model_tester.prepare_config_and_inputs_for_common()
			for model_class in self.all_model_classes[:2]:
				A							  =      model_class(_lowerCAmelCase      )
				if getattr(_lowerCAmelCase      ,      """hf_compute_loss"""      ,      _lowerCAmelCase      ):
					# Test that model correctly compute the loss with kwargs
					A							  =      self._prepare_for_class(inputs_dict.copy()      ,      _lowerCAmelCase      ,      return_labels=_lowerCAmelCase      )
					# Is there a better way to remove these decoder inputs?
					A							  =      {
					    key: val
					    for key, val in prepared_for_class.items()
					    if key not in ("""head_mask""", """decoder_head_mask""", """cross_attn_head_mask""", """decoder_input_ids""")
					}

					A							  =      {
					    """labels""",
					    """label""",
					    """label_ids""",
					    """start_positions""",
					    """start_position""",
					    """end_positions""",
					    """end_position""",
					    """next_sentence_label""",
					}
					A							  =      possible_label_cols.intersection(set(_lowerCAmelCase      )      )
					self.assertGreater(len(_lowerCAmelCase      )      ,      0      ,      msg="""No matching label names found!"""      )
					A							  =      {key: val for key, val in prepared_for_class.items() if key in label_names}
					A							  =      {key: val for key, val in prepared_for_class.items() if key not in label_names}
					self.assertGreater(len(_lowerCAmelCase      )      ,      0      )

					# Make sure it works with XLA!
					model.compile(optimizer=tf.keras.optimizers.SGD(0.0      )      ,      jit_compile=_lowerCAmelCase      )
					# Make sure the model fits without crashing regardless of where we pass the labels
					A							  =      model.fit(
					    _lowerCAmelCase      ,      validation_data=_lowerCAmelCase      ,      steps_per_epoch=1      ,      validation_steps=1      ,      shuffle=_lowerCAmelCase      ,      verbose=0      ,      )
					A							  =      history.history["""loss"""][0]
					self.assertTrue(not isnan(_lowerCAmelCase      )      )
					A							  =      history.history["""val_loss"""][0]
					self.assertTrue(not isnan(_lowerCAmelCase      )      )

					# Now test it with separate labels, to make sure that path works in XLA too.
					A							  =      model_class(_lowerCAmelCase      )
					model.compile(optimizer=tf.keras.optimizers.SGD(0.0      )      ,      jit_compile=_lowerCAmelCase      )
					A							  =      model.fit(
					    _lowerCAmelCase      ,      _lowerCAmelCase      ,      validation_data=(inputs_minus_labels, labels)      ,      steps_per_epoch=1      ,      validation_steps=1      ,      shuffle=_lowerCAmelCase      ,      verbose=0      ,      )

					A							  =      history.history["""loss"""][0]
					self.assertTrue(not isnan(_lowerCAmelCase      )      )
					A							  =      history.history["""val_loss"""][0]
					self.assertTrue(not isnan(_lowerCAmelCase      )      )







		@slow
		def        A     (self						:      Union[str, Any]      ):
			A  ,    A							  =      self.model_tester.prepare_config_and_inputs_for_common()
			A							  =      True
			A							  =      True

			if hasattr(_lowerCAmelCase      ,      """use_cache"""      ):
				A							  =      True

			A							  =      getattr(self.model_tester      ,      """encoder_seq_length"""      ,      self.model_tester.seq_length      )
			A							  =      getattr(self.model_tester      ,      """key_length"""      ,      _lowerCAmelCase      )

			for model_class in self.all_model_classes[:2]:
				A							  =      self._prepare_for_class(_lowerCAmelCase      ,      _lowerCAmelCase      )
				A							  =      model_class(_lowerCAmelCase      )
				model.build()
				A							  =      len(model(_lowerCAmelCase      )      )

				for key in list(class_inputs_dict.keys()      ):
					# Remove keys not in the serving signature, as the SavedModel will not be compiled to deal with them
					if key not in model.input_signature:
						del class_inputs_dict[key]
					# Check it's a tensor, in case the inputs dict has some bools in it too
					elif isinstance(class_inputs_dict[key]      ,      tf.Tensor      ) and class_inputs_dict[key].dtype.is_integer:
						A							  =      tf.cast(class_inputs_dict[key]      ,      tf.intaa      )

				if set(class_inputs_dict.keys()      ) != set(model.input_signature.keys()      ):
					continue  # Some models have inputs that the preparation functions don't create, we skip those

				with tempfile.TemporaryDirectory() as tmpdirname:
					model.save_pretrained(_lowerCAmelCase      ,      saved_model=_lowerCAmelCase      )
					A							  =      os.path.join(_lowerCAmelCase      ,      """saved_model"""      ,      """1"""      )
					A							  =      tf.keras.models.load_model(_lowerCAmelCase      )
					A							  =      model(_lowerCAmelCase      )

					if self.is_encoder_decoder:
						A							  =      outputs["""encoder_hidden_states"""]
						A							  =      outputs["""encoder_attentions"""]
					else:
						A							  =      outputs["""hidden_states"""]
						A							  =      outputs["""attentions"""]

					self.assertEqual(len(_lowerCAmelCase      )      ,      _lowerCAmelCase      )

					A							  =      getattr(
					    self.model_tester      ,      """expected_num_hidden_layers"""      ,      self.model_tester.num_hidden_layers + 1      )

					self.assertEqual(len(_lowerCAmelCase      )      ,      _lowerCAmelCase      )
					self.assertListEqual(
					    list(output_hidden_states[0].shape[-2:]      )      ,      [self.model_tester.seq_length, self.model_tester.hidden_size]      ,      )

					self.assertEqual(len(_lowerCAmelCase      )      ,      self.model_tester.num_hidden_layers      )
					self.assertListEqual(
					    list(output_attentions[0].shape[-3:]      )      ,      [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]      ,      )







		@slow
		def        A     (self						:      Dict      ):
			tf.keras.mixed_precision.set_global_policy("""mixed_float16"""      )

			# try/finally block to ensure subsequent tests run in float32
			try:
				A  ,    A							  =      self.model_tester.prepare_config_and_inputs_for_common()
				for model_class in self.all_model_classes[:2]:
					A							  =      self._prepare_for_class(_lowerCAmelCase      ,      _lowerCAmelCase      )
					A							  =      model_class(_lowerCAmelCase      )
					A							  =      model(_lowerCAmelCase      )

					self.assertIsNotNone(_lowerCAmelCase      )
			finally:
				tf.keras.mixed_precision.set_global_policy("""float32"""      )







		@slow
		def        A     (self						:      int      ):
			A  ,    A							  =      self.model_tester.prepare_config_and_inputs_for_common()
			# head_mask and decoder_head_mask has different shapes than other input args
			if "head_mask" in inputs_dict:
				del inputs_dict["head_mask"]
			if "decoder_head_mask" in inputs_dict:
				del inputs_dict["decoder_head_mask"]
			if "cross_attn_head_mask" in inputs_dict:
				del inputs_dict["cross_attn_head_mask"]
			A							  =      {
			    module_member
			    for model_class in self.all_model_classes
			    for module in (import_module(model_class.__module__      ),)
			    for module_member_name in dir(_lowerCAmelCase      )
			    if module_member_name.endswith("""MainLayer"""      )
			    for module_member in (getattr(_lowerCAmelCase      ,      _lowerCAmelCase      ),)
			    if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      )
			    and tf.keras.layers.Layer in module_member.__bases__
			    and getattr(_lowerCAmelCase      ,      """_keras_serializable"""      ,      _lowerCAmelCase      )
			}

			for main_layer_class in tf_main_layer_classes:
				# T5MainLayer needs an embed_tokens parameter when called without the inputs_embeds parameter
				if "T5" in main_layer_class.__name__:
					# Take the same values than in TFT5ModelTester for this shared layer
					A							  =      TFSharedEmbeddings(self.model_tester.vocab_size      ,      self.model_tester.hidden_size      ,      name="""shared"""      )
					A							  =      False
					A							  =      main_layer_class(_lowerCAmelCase      ,      embed_tokens=_lowerCAmelCase      )
				else:
					A							  =      main_layer_class(_lowerCAmelCase      )

				A							  =      {
				    name: tf.keras.Input(tensor.shape[1:]      ,      dtype=tensor.dtype      ) for name, tensor in inputs_dict.items()
				}

				if hasattr(self.model_tester      ,      """num_labels"""      ):
					A							  =      self.model_tester.num_labels
				else:
					A							  =      2

				A							  =      tf.data.Dataset.from_tensor_slices(
				    (inputs_dict, np.ones((self.model_tester.batch_size, self.model_tester.seq_length, num_labels, 1)      ))      ).batch(1      )

				A							  =      main_layer(_lowerCAmelCase      )[0]
				A							  =      tf.keras.layers.Dense(_lowerCAmelCase      ,      activation="""softmax"""      ,      name="""outputs"""      )(_lowerCAmelCase      )
				A							  =      tf.keras.models.Model(inputs=_lowerCAmelCase      ,      outputs=[outputs]      )

				model.compile(loss="""binary_crossentropy"""      ,      optimizer="""adam"""      ,      metrics=["""binary_accuracy"""]      )
				model.fit(_lowerCAmelCase      ,      epochs=1      )

				with tempfile.TemporaryDirectory() as tmpdirname:
					A							  =      os.path.join(_lowerCAmelCase      ,      """keras_model.h5"""      )
					model.save(_lowerCAmelCase      )
					if "T5" in main_layer_class.__name__:
						A							  =      tf.keras.models.load_model(
						    _lowerCAmelCase      ,      custom_objects={
						        main_layer_class.__name__: main_layer_class,
						        """TFSharedEmbeddings""": TFSharedEmbeddings,
						    }      ,      )
					else:
						A							  =      tf.keras.models.load_model(
						    _lowerCAmelCase      ,      custom_objects={main_layer_class.__name__: main_layer_class}      )
					assert isinstance(_lowerCAmelCase      ,      tf.keras.Model      )
					model(_lowerCAmelCase      )







		@slow
		def        A     (self						:      int      ):
			A  ,    A							  =      self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes[:2]:
				A							  =      model_class(_lowerCAmelCase      )

				A							  =      copy.deepcopy(_lowerCAmelCase      )

				if not self.is_encoder_decoder:
					A							  =      inputs["""input_ids"""]
					del inputs["input_ids"]
				else:
					A							  =      inputs["""input_ids"""]
					A							  =      inputs.get("""decoder_input_ids"""      ,      _lowerCAmelCase      )
					del inputs["input_ids"]
					inputs.pop("""decoder_input_ids"""      ,      _lowerCAmelCase      )

				if not self.is_encoder_decoder:
					A							  =      model.get_input_embeddings()(_lowerCAmelCase      )
				else:
					A							  =      model.get_input_embeddings()(_lowerCAmelCase      )
					A							  =      model.get_input_embeddings()(_lowerCAmelCase      )

				A							  =      self._prepare_for_class(_lowerCAmelCase      ,      _lowerCAmelCase      )

				@tf.function
				def run_in_graph_mode():
					return model(_lowerCAmelCase      )

				A							  =      run_in_graph_mode()
				self.assertIsNotNone(_lowerCAmelCase      )







		def        A     (self						:      Union[str, Any]      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      Tuple      ):
			# special tokens cannot be bad tokens
			A							  =      []
			if model.config.bos_token_id is not None:
				special_tokens.append(model.config.bos_token_id      )
			if model.config.pad_token_id is not None:
				special_tokens.append(model.config.pad_token_id      )
			if model.config.eos_token_id is not None:
				special_tokens.append(model.config.eos_token_id      )

			# create random bad tokens that are not special tokens
			A							  =      []
			while len(_lowerCAmelCase      ) < num_bad_tokens:
				A							  =      tf.squeeze(ids_tensor((1, 1)      ,      self.model_tester.vocab_size      )      ,      0      ).numpy()[0]
				if token not in special_tokens:
					bad_tokens.append(_lowerCAmelCase      )
			return bad_tokens







		def        A     (self						:      Tuple      ,      _lowerCAmelCase						:      Optional[int]      ):
			for token_id in output_ids[0].numpy().tolist():
				self.assertGreaterEqual(_lowerCAmelCase      ,      0      )
				self.assertLess(_lowerCAmelCase      ,      self.model_tester.vocab_size      )







		def        A     (self						:      Optional[int]      ,      _lowerCAmelCase						:      Optional[int]      ,      _lowerCAmelCase						:      List[Any]      ):
			# for all bad word tokens
			for bad_word_ids in bad_words_ids:
				# for all slices in batch
				for generated_ids_slice in generated_ids:
					# for all word idx
					for i in range(len(_lowerCAmelCase      )      ,      len(_lowerCAmelCase      )      ):
						# if tokens match
						if generated_ids_slice[i - len(_lowerCAmelCase      ) : i] == bad_word_ids:
							return True
			return False








'''simple docstring'''


import tempfile
import unittest

import numpy as np
import torch

from diffusers import DiffusionPipeline
from diffusers.models.attention_processor import Attention, AttnAddedKVProcessor



class 							__UpperCAmelCase  (       unittest.TestCase						):







		'''simple docstring'''







		def        A     (self						:      Optional[int]      ,      _lowerCAmelCase						:      bool = False      ):
			A							  =      10

			if only_cross_attention:
				A							  =      12
			else:
				# when only cross attention is not set, the cross attention dim must be the same as the query dim
				A							  =      query_dim

			return {
			    "query_dim": query_dim,
			    "cross_attention_dim": cross_attention_dim,
			    "heads": 2,
			    "dim_head": 4,
			    "added_kv_proj_dim": 6,
			    "norm_num_groups": 1,
			    "only_cross_attention": only_cross_attention,
			    "processor": AttnAddedKVProcessor(),
			}







		def        A     (self						:      Optional[int]      ,      _lowerCAmelCase						:      Dict      ,      _lowerCAmelCase						:      Any      ):
			A							  =      2

			A							  =      torch.rand(_lowerCAmelCase      ,      _lowerCAmelCase      ,      3      ,      2      )
			A							  =      torch.rand(_lowerCAmelCase      ,      4      ,      _lowerCAmelCase      )
			A							  =      None

			return {
			    "hidden_states": hidden_states,
			    "encoder_hidden_states": encoder_hidden_states,
			    "attention_mask": attention_mask,
			}







		def        A     (self						:      Optional[Any]      ):
			# self and cross attention

			torch.manual_seed(0      )

			A							  =      self.get_constructor_arguments(only_cross_attention=_lowerCAmelCase      )
			A							  =      Attention(**_lowerCAmelCase      )

			self.assertTrue(attn.to_k is not None      )
			self.assertTrue(attn.to_v is not None      )

			A							  =      self.get_forward_arguments(
			    query_dim=constructor_args["""query_dim"""]      ,      added_kv_proj_dim=constructor_args["""added_kv_proj_dim"""]      )

			A							  =      attn(**_lowerCAmelCase      )

			# only self attention

			torch.manual_seed(0      )

			A							  =      self.get_constructor_arguments(only_cross_attention=_lowerCAmelCase      )
			A							  =      Attention(**_lowerCAmelCase      )

			self.assertTrue(attn.to_k is None      )
			self.assertTrue(attn.to_v is None      )

			A							  =      self.get_forward_arguments(
			    query_dim=constructor_args["""query_dim"""]      ,      added_kv_proj_dim=constructor_args["""added_kv_proj_dim"""]      )

			A							  =      attn(**_lowerCAmelCase      )

			self.assertTrue((only_cross_attn_out != self_and_cross_attn_out).all()      )






class 							__UpperCAmelCase  (       unittest.TestCase						):







		'''simple docstring'''







		def        A     (self						:      Any      ):
			A							  =      DiffusionPipeline.from_pretrained("""hf-internal-testing/tiny-stable-diffusion-pipe"""      ,      safety_checker=_lowerCAmelCase      )

			A							  =      pipe(
			    """foo"""      ,      num_inference_steps=2      ,      generator=torch.Generator("""cpu"""      ).manual_seed(0      )      ,      output_type="""np"""      ,      ).images

			# the initial conversion succeeds
			A							  =      DiffusionPipeline.from_pretrained(
			    """hf-internal-testing/tiny-stable-diffusion-pipe"""      ,      device_map="""sequential"""      ,      safety_checker=_lowerCAmelCase      )

			A							  =      pipe(
			    """foo"""      ,      num_inference_steps=2      ,      generator=torch.Generator("""cpu"""      ).manual_seed(0      )      ,      output_type="""np"""      ,      ).images

			with tempfile.TemporaryDirectory() as tmpdir:
				# save the converted model
				pipe.save_pretrained(_lowerCAmelCase      )

				# can also load the converted weights
				A							  =      DiffusionPipeline.from_pretrained(_lowerCAmelCase      ,      device_map="""sequential"""      ,      safety_checker=_lowerCAmelCase      )

			A							  =      pipe(
			    """foo"""      ,      num_inference_steps=2      ,      generator=torch.Generator("""cpu"""      ).manual_seed(0      )      ,      output_type="""np"""      ,      ).images

			self.assertTrue(np.allclose(_lowerCAmelCase      ,      _lowerCAmelCase      ,      atol=1e-5      )      )
			self.assertTrue(np.allclose(_lowerCAmelCase      ,      _lowerCAmelCase      ,      atol=1e-5      )      )

