






'''simple docstring'''







from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel

@require_tf
class _lowercase      :
							a							 =			MarianConfig
							a							 =			{}
							a							 =			"""gelu"""




							def __init__(				self:							Any     ,				UpperCamelCase__:							Optional[int]     ,				UpperCamelCase__:							int=13     ,				UpperCamelCase__:							Union[str, Any]=7     ,				UpperCamelCase__:							int=True     ,				UpperCamelCase__:							Optional[Any]=False     ,				UpperCamelCase__:							Optional[int]=99     ,				UpperCamelCase__:							Dict=32     ,				UpperCamelCase__:							List[Any]=2     ,				UpperCamelCase__:							List[str]=4     ,				UpperCamelCase__:							Optional[Any]=37     ,				UpperCamelCase__:							List[Any]=0.1     ,				UpperCamelCase__:							Union[str, Any]=0.1     ,				UpperCamelCase__:							int=20     ,				UpperCamelCase__:							Dict=2     ,				UpperCamelCase__:							Dict=1     ,				UpperCamelCase__:							str=0     ,				):
														lowerCamelCase__   :      Any			    =			parent
														lowerCamelCase__   :      List[str]			    =			batch_size
														lowerCamelCase__   :      List[Any]			    =			seq_length
														lowerCamelCase__   :      List[Any]			    =			is_training
														lowerCamelCase__   :      Optional[int]			    =			use_labels
														lowerCamelCase__   :      List[Any]			    =			vocab_size
														lowerCamelCase__   :      Optional[int]			    =			hidden_size
														lowerCamelCase__   :      Optional[Any]			    =			num_hidden_layers
														lowerCamelCase__   :      str			    =			num_attention_heads
														lowerCamelCase__   :      Dict			    =			intermediate_size

														lowerCamelCase__   :      List[str]			    =			hidden_dropout_prob
														lowerCamelCase__   :      Union[str, Any]			    =			attention_probs_dropout_prob
														lowerCamelCase__   :      Optional[Any]			    =			max_position_embeddings
														lowerCamelCase__   :      Tuple			    =			eos_token_id
														lowerCamelCase__   :      Any			    =			pad_token_id
														lowerCamelCase__   :      Union[str, Any]			    =			bos_token_id




							def 						lowerCamelCase_							(				self:							List[str]	):
														lowerCamelCase__   :      Union[str, Any]			    =			ids_tensor([self.batch_size, self.seq_length - 1]     ,				self.vocab_size	)
														lowerCamelCase__   :      List[str]			    =			tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size	)     ,				1	)
														lowerCamelCase__   :      str			    =			tf.concat([input_ids, eos_tensor]     ,				axis=1	)

														lowerCamelCase__   :      Dict			    =			ids_tensor([self.batch_size, self.seq_length]     ,				self.vocab_size	)

														lowerCamelCase__   :      List[Any]			    =			self.config_cls(
														    vocab_size=self.vocab_size     ,				d_model=self.hidden_size     ,				encoder_layers=self.num_hidden_layers     ,				decoder_layers=self.num_hidden_layers     ,				encoder_attention_heads=self.num_attention_heads     ,				decoder_attention_heads=self.num_attention_heads     ,				encoder_ffn_dim=self.intermediate_size     ,				decoder_ffn_dim=self.intermediate_size     ,				dropout=self.hidden_dropout_prob     ,				attention_dropout=self.attention_probs_dropout_prob     ,				max_position_embeddings=self.max_position_embeddings     ,				eos_token_ids=[2]     ,				bos_token_id=self.bos_token_id     ,				pad_token_id=self.pad_token_id     ,				decoder_start_token_id=self.pad_token_id     ,				**self.config_updates     ,				)
														lowerCamelCase__   :      List[str]			    =			prepare_marian_inputs_dict(UpperCamelCase__     ,				UpperCamelCase__     ,				UpperCamelCase__	)
														return config, inputs_dict




							def 						lowerCamelCase_							(				self:							List[str]     ,				UpperCamelCase__:							List[Any]     ,				UpperCamelCase__:							Optional[int]	):
														lowerCamelCase__   :      Optional[Any]			    =			TFMarianModel(config=UpperCamelCase__	).get_decoder()
														lowerCamelCase__   :      List[str]			    =			inputs_dict["""input_ids"""]

														lowerCamelCase__   :      List[str]			    =			input_ids[:1, :]
														lowerCamelCase__   :      Tuple			    =			inputs_dict["""attention_mask"""][:1, :]
														lowerCamelCase__   :      Union[str, Any]			    =			inputs_dict["""head_mask"""]
														lowerCamelCase__   :      Union[str, Any]			    =			1

														# first forward pass
														lowerCamelCase__   :      Dict			    =			model(UpperCamelCase__     ,				attention_mask=UpperCamelCase__     ,				head_mask=UpperCamelCase__     ,				use_cache=UpperCamelCase__	)

														lowerCamelCase__    ,						lowerCamelCase__   :      Optional[Any]			    =			outputs.to_tuple()

														# create hypothetical next token and extent to next_input_ids
														lowerCamelCase__   :      Any			    =			ids_tensor((self.batch_size, 3)     ,				config.vocab_size	)
														lowerCamelCase__   :      Tuple			    =			tf.cast(ids_tensor((self.batch_size, 3)     ,				2	)     ,				tf.inta	)

														# append to next input_ids and
														lowerCamelCase__   :      Tuple			    =			tf.concat([input_ids, next_tokens]     ,				axis=-1	)
														lowerCamelCase__   :      List[str]			    =			tf.concat([attention_mask, next_attn_mask]     ,				axis=-1	)

														lowerCamelCase__   :      int			    =			model(UpperCamelCase__     ,				attention_mask=UpperCamelCase__	)[0]
														lowerCamelCase__   :      Union[str, Any]			    =			model(UpperCamelCase__     ,				attention_mask=UpperCamelCase__     ,				past_key_values=UpperCamelCase__	)[0]

														self.parent.assertEqual(next_tokens.shape[1]     ,				output_from_past.shape[1]	)

														# select random slice
														lowerCamelCase__   :      Dict			    =			int(ids_tensor((1,)     ,				output_from_past.shape[-1]	)	)
														lowerCamelCase__   :      Any			    =			output_from_no_past[:, -3:, random_slice_idx]
														lowerCamelCase__   :      Any			    =			output_from_past[:, :, random_slice_idx]

														# test that outputs are equal for slice
														tf.debugging.assert_near(UpperCamelCase__     ,				UpperCamelCase__     ,				rtol=1e-3	)





def 					SCREAMING_SNAKE_CASE_			(UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase=None							,     UpperCamelCase=None							,     UpperCamelCase=None							,     UpperCamelCase=None							,     UpperCamelCase=None							,     ) ->			List[Any]:
							if attention_mask is None:
														lowerCamelCase__   :      Optional[int]			    =			tf.cast(tf.math.not_equal(UpperCamelCase							,     config.pad_token_id )							,     tf.inta )
							if decoder_attention_mask is None:
														lowerCamelCase__   :      Optional[int]			    =			tf.concat(
														    [
														        tf.ones(decoder_input_ids[:, :1].shape							,     dtype=tf.inta ),
														        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]							,     config.pad_token_id )							,     tf.inta ),
														    ]							,     axis=-1							,     )
							if head_mask is None:
														lowerCamelCase__   :      List[Any]			    =			tf.ones((config.encoder_layers, config.encoder_attention_heads) )
							if decoder_head_mask is None:
														lowerCamelCase__   :      Tuple			    =			tf.ones((config.decoder_layers, config.decoder_attention_heads) )
							if cross_attn_head_mask is None:
														lowerCamelCase__   :      Dict			    =			tf.ones((config.decoder_layers, config.decoder_attention_heads) )
							return {
							    "input_ids": input_ids,
							    "decoder_input_ids": decoder_input_ids,
							    "attention_mask": attention_mask,
							    "decoder_attention_mask": decoder_attention_mask,
							    "head_mask": head_mask,
							    "decoder_head_mask": decoder_head_mask,
							    "cross_attn_head_mask": cross_attn_head_mask,
							}

@require_tf
class _lowercase      (							_lowercase							, _lowercase							, unittest.TestCase						):
							a							 =			(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
							a							 =			(TFMarianMTModel,) if is_tf_available() else ()
							a							 =			(
							    {
							        """conversational""": TFMarianMTModel,
							        """feature-extraction""": TFMarianModel,
							        """summarization""": TFMarianMTModel,
							        """text2text-generation""": TFMarianMTModel,
							        """translation""": TFMarianMTModel,
							    }
							    if is_tf_available()
							    else {}
							)
							a							 =			True
							a							 =			False
							a							 =			False




							def 						lowerCamelCase_							(				self:							Optional[Any]	):
														lowerCamelCase__   :      Optional[Any]			    =			TFMarianModelTester(self	)
														lowerCamelCase__   :      Optional[Any]			    =			ConfigTester(self     ,				config_class=UpperCamelCase__	)




							def 						lowerCamelCase_							(				self:							List[str]	):
														self.config_tester.run_common_tests()




							def 						lowerCamelCase_							(				self:							Optional[int]	):
														lowerCamelCase__   :      Union[str, Any]			    =			self.model_tester.prepare_config_and_inputs_for_common()
														self.model_tester.check_decoder_model_past_large_inputs(*UpperCamelCase__	)



@require_tf
class _lowercase      (							unittest.TestCase						):
							a							 =			1000  # show more chars for failing integration tests




							@classmethod
							def 						lowerCamelCase_							(				cls:							Tuple	):
														lowerCamelCase__   :      Optional[Any]			    =			F'''Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}'''
														return cls




							@cached_property
							def 						lowerCamelCase_							(				self:							Optional[Any]	):
														return AutoTokenizer.from_pretrained(self.model_name	)




							@property
							def 						lowerCamelCase_							(				self:							Dict	):
														return self.tokenizer.eos_token_id




							@cached_property
							def 						lowerCamelCase_							(				self:							List[Any]	):
														warnings.simplefilter("""error"""	)
														lowerCamelCase__   :      TFMarianMTModel			    =			TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name	)
														assert isinstance(UpperCamelCase__     ,				UpperCamelCase__	)
														lowerCamelCase__   :      Optional[Any]			    =			model.config
														self.assertListEqual(c.bad_words_ids     ,				[[c.pad_token_id]]	)
														self.assertEqual(c.max_length     ,				512	)
														self.assertEqual(c.decoder_start_token_id     ,				c.pad_token_id	)
														return model




							def 						lowerCamelCase_							(				self:							List[Any]     ,				**UpperCamelCase__:							Union[str, Any]	):
														lowerCamelCase__   :      List[str]			    =			self.translate_src_text(**UpperCamelCase__	)
														self.assertListEqual(self.expected_text     ,				UpperCamelCase__	)




							def 						lowerCamelCase_							(				self:							Dict     ,				**UpperCamelCase__:							List[Any]	):
														lowerCamelCase__   :      List[Any]			    =			self.tokenizer(self.src_text     ,				**UpperCamelCase__     ,				padding=UpperCamelCase__     ,				return_tensors="""tf"""	)
														lowerCamelCase__   :      Tuple			    =			self.model.generate(
														    model_inputs.input_ids     ,				attention_mask=model_inputs.attention_mask     ,				num_beams=2     ,				max_length=128	)
														lowerCamelCase__   :      Union[str, Any]			    =			self.tokenizer.batch_decode(generated_ids.numpy()     ,				skip_special_tokens=UpperCamelCase__	)
														return generated_words



@require_sentencepiece
@require_tokenizers
@require_tf
class _lowercase      (							_lowercase						):
							a							 =			"""mt"""
							a							 =			"""en"""
							a							 =			["""Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."""]
							a							 =			["""Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."""]




							@unittest.skip("""Skipping until #12647 is resolved."""	)
							@slow
							def 						lowerCamelCase_							(				self:							List[Any]	):
														self._assert_generated_batch_equal_expected()



@require_sentencepiece
@require_tokenizers
@require_tf
class _lowercase      (							_lowercase						):
							a							 =			"""en"""
							a							 =			"""zh"""
							a							 =			["""My name is Wolfgang and I live in Berlin"""]
							a							 =			["""我叫沃尔夫冈 我住在柏林"""]




							@unittest.skip("""Skipping until #12647 is resolved."""	)
							@slow
							def 						lowerCamelCase_							(				self:							Optional[Any]	):
														self._assert_generated_batch_equal_expected()



@require_sentencepiece
@require_tokenizers
@require_tf
class _lowercase      (							_lowercase						):
							a							 =			"""en"""
							a							 =			"""ROMANCE"""
							a							 =			[
							    """>>fr<< Don't spend so much time watching TV.""",
							    """>>pt<< Your message has been sent.""",
							    """>>es<< He's two years older than me.""",
							]
							a							 =			[
							    """Ne passez pas autant de temps à regarder la télé.""",
							    """A sua mensagem foi enviada.""",
							    """Es dos años más viejo que yo.""",
							]




							@unittest.skip("""Skipping until #12647 is resolved."""	)
							@slow
							def 						lowerCamelCase_							(				self:							Any	):
														self._assert_generated_batch_equal_expected()




							@unittest.skip("""Skipping until #12647 is resolved."""	)
							@slow
							def 						lowerCamelCase_							(				self:							str	):
														lowerCamelCase__   :      Optional[Any]			    =			TranslationPipeline(self.model     ,				self.tokenizer     ,				framework="""tf"""	)
														lowerCamelCase__   :      Optional[int]			    =			pipeline(self.src_text	)
														self.assertEqual(self.expected_text     ,				[x["""translation_text"""] for x in output]	)









'''simple docstring'''







import unittest
from pathlib import Path

from transformers import is_vision_available, load_tool
from transformers.testing_utils import get_tests_dir

from .test_tools_common import ToolTesterMixin


if is_vision_available():
			from PIL import Image

class _lowercase      (							unittest.TestCase							, _lowercase						):




							def 						lowerCamelCase_							(				self:							Tuple	):
														lowerCamelCase__   :      Optional[Any]			    =			load_tool("""image-captioning"""	)
														self.tool.setup()
														lowerCamelCase__   :      Optional[Any]			    =			load_tool("""image-captioning"""     ,				remote=UpperCamelCase__	)




							def 						lowerCamelCase_							(				self:							int	):
														lowerCamelCase__   :      List[Any]			    =			Image.open(Path(get_tests_dir("""fixtures/tests_samples/COCO"""	)	) / """000000039769.png"""	)
														lowerCamelCase__   :      Optional[Any]			    =			self.tool(UpperCamelCase__	)
														self.assertEqual(UpperCamelCase__     ,				"""two cats sleeping on a couch"""	)




							def 						lowerCamelCase_							(				self:							Union[str, Any]	):
														lowerCamelCase__   :      Tuple			    =			Image.open(Path(get_tests_dir("""fixtures/tests_samples/COCO"""	)	) / """000000039769.png"""	)
														lowerCamelCase__   :      Dict			    =			self.remote_tool(UpperCamelCase__	)
														self.assertEqual(UpperCamelCase__     ,				"""two cats sleeping on a couch"""	)




							def 						lowerCamelCase_							(				self:							str	):
														lowerCamelCase__   :      Optional[int]			    =			Image.open(Path(get_tests_dir("""fixtures/tests_samples/COCO"""	)	) / """000000039769.png"""	)
														lowerCamelCase__   :      str			    =			self.tool(image=UpperCamelCase__	)
														self.assertEqual(UpperCamelCase__     ,				"""two cats sleeping on a couch"""	)




							def 						lowerCamelCase_							(				self:							Union[str, Any]	):
														lowerCamelCase__   :      Optional[int]			    =			Image.open(Path(get_tests_dir("""fixtures/tests_samples/COCO"""	)	) / """000000039769.png"""	)
														lowerCamelCase__   :      Optional[int]			    =			self.remote_tool(image=UpperCamelCase__	)
														self.assertEqual(UpperCamelCase__     ,				"""two cats sleeping on a couch"""	)

