



from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel
@require_tf
class lowerCAmelCase_       :







	'''simple docstring'''




	A_       :     str									=		MarianConfig
	A_       :     Optional[int]									=		{}
	A_       :     Dict									=		"""gelu"""

	def __init__(	self   ,       lowerCamelCase   ,       lowerCamelCase=13   ,       lowerCamelCase=7   ,       lowerCamelCase=True   ,       lowerCamelCase=False   ,       lowerCamelCase=99   ,       lowerCamelCase=32   ,       lowerCamelCase=2   ,       lowerCamelCase=4   ,       lowerCamelCase=37   ,       lowerCamelCase=0.1   ,       lowerCamelCase=0.1   ,       lowerCamelCase=20   ,       lowerCamelCase=2   ,       lowerCamelCase=1   ,       lowerCamelCase=0   ,       ):



				'''simple docstring'''


				a__					=  parent
				a__					=  batch_size
				a__					=  seq_length
				a__					=  is_training
				a__					=  use_labels
				a__					=  vocab_size
				a__					=  hidden_size
				a__					=  num_hidden_layers
				a__					=  num_attention_heads
				a__					=  intermediate_size

				a__					=  hidden_dropout_prob
				a__					=  attention_probs_dropout_prob
				a__					=  max_position_embeddings
				a__					=  eos_token_id
				a__					=  pad_token_id
				a__					=  bos_token_id

	def 				_A			(	self       ):



				'''simple docstring'''


				a__					=  ids_tensor([self.batch_size, self.seq_length - 1]   ,       self.vocab_size       )
				a__					=  tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size       )   ,       1       )
				a__					=  tf.concat([input_ids, eos_tensor]   ,       axis=1       )

				a__					=  ids_tensor([self.batch_size, self.seq_length]   ,       self.vocab_size       )

				a__					=  self.config_cls(
				    vocab_size=self.vocab_size   ,       d_model=self.hidden_size   ,       encoder_layers=self.num_hidden_layers   ,       decoder_layers=self.num_hidden_layers   ,       encoder_attention_heads=self.num_attention_heads   ,       decoder_attention_heads=self.num_attention_heads   ,       encoder_ffn_dim=self.intermediate_size   ,       decoder_ffn_dim=self.intermediate_size   ,       dropout=self.hidden_dropout_prob   ,       attention_dropout=self.attention_probs_dropout_prob   ,       max_position_embeddings=self.max_position_embeddings   ,       eos_token_ids=[2]   ,       bos_token_id=self.bos_token_id   ,       pad_token_id=self.pad_token_id   ,       decoder_start_token_id=self.pad_token_id   ,       **self.config_updates   ,       )
				a__					=  prepare_marian_inputs_dict(snake_case_   ,       snake_case_   ,       snake_case_       )
				return config, inputs_dict







	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase       ):



				'''simple docstring'''


				a__					=  TFMarianModel(config=snake_case_       ).get_decoder()
				a__					=  inputs_dict["input_ids"]

				a__					=  input_ids[:1, :]
				a__					=  inputs_dict["attention_mask"][:1, :]
				a__					=  inputs_dict["head_mask"]
				a__					=  1

				# first forward pass
				a__					=  model(snake_case_   ,       attention_mask=snake_case_   ,       head_mask=snake_case_   ,       use_cache=snake_case_       )

				a__					=  outputs.to_tuple()

				# create hypothetical next token and extent to next_input_ids
				a__					=  ids_tensor((self.batch_size, 3)   ,       config.vocab_size       )
				a__					=  tf.cast(ids_tensor((self.batch_size, 3)   ,       2       )   ,       tf.inta       )

				# append to next input_ids and
				a__					=  tf.concat([input_ids, next_tokens]   ,       axis=-1       )
				a__					=  tf.concat([attention_mask, next_attn_mask]   ,       axis=-1       )

				a__					=  model(snake_case_   ,       attention_mask=snake_case_       )[0]
				a__					=  model(snake_case_   ,       attention_mask=snake_case_   ,       past_key_values=snake_case_       )[0]

				self.parent.assertEqual(next_tokens.shape[1]   ,       output_from_past.shape[1]       )

				# select random slice
				a__					=  int(ids_tensor((1,)   ,       output_from_past.shape[-1]       )       )
				a__					=  output_from_no_past[:, -3:, random_slice_idx]
				a__					=  output_from_past[:, :, random_slice_idx]

				# test that outputs are equal for slice
				tf.debugging.assert_near(snake_case_   ,       snake_case_   ,       rtol=1e-3       )




def    UpperCAmelCase					(  lowercase__						:					Union[str, Any]				,					lowercase__						:					Dict				,					lowercase__						:					str				,					lowercase__						:					Optional[Any]=None				,					lowercase__						:					int=None				,					lowercase__						:					Tuple=None				,					lowercase__						:					Optional[Any]=None				,					lowercase__						:					List[str]=None				,					):



			'''simple docstring'''





			if attention_mask is None:
						a__					=  tf.cast(tf.math.not_equal(_lowerCAmelCase				,					config.pad_token_id	)				,					tf.inta	)
			if decoder_attention_mask is None:
						a__					=  tf.concat(
						    [
						        tf.ones(decoder_input_ids[:, :1].shape				,					dtype=tf.inta	),
						        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]				,					config.pad_token_id	)				,					tf.inta	),
						    ]				,					axis=-1				,					)
			if head_mask is None:
						a__					=  tf.ones((config.encoder_layers, config.encoder_attention_heads)	)
			if decoder_head_mask is None:
						a__					=  tf.ones((config.decoder_layers, config.decoder_attention_heads)	)
			if cross_attn_head_mask is None:
						a__					=  tf.ones((config.decoder_layers, config.decoder_attention_heads)	)
			return {
			    "input_ids": input_ids,
			    "decoder_input_ids": decoder_input_ids,
			    "attention_mask": attention_mask,
			    "decoder_attention_mask": decoder_attention_mask,
			    "head_mask": head_mask,
			    "decoder_head_mask": decoder_head_mask,
			    "cross_attn_head_mask": cross_attn_head_mask,
			}
@require_tf
class lowerCAmelCase_       (   _UpperCAmelCase ,_UpperCAmelCase ,unittest.TestCase			):







	'''simple docstring'''




	A_       :     Optional[int]									=		(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
	A_       :     Tuple									=		(TFMarianMTModel,) if is_tf_available() else ()
	A_       :     Dict									=		(
	    {
	        """conversational""": TFMarianMTModel,
	        """feature-extraction""": TFMarianModel,
	        """summarization""": TFMarianMTModel,
	        """text2text-generation""": TFMarianMTModel,
	        """translation""": TFMarianMTModel,
	    }
	    if is_tf_available()
	    else {}
	)
	A_       :     List[Any]									=		True
	A_       :     List[Any]									=		False
	A_       :     Optional[Any]									=		False

	def 				_A			(	self       ):



				'''simple docstring'''


				a__					=  TFMarianModelTester(self       )
				a__					=  ConfigTester(self   ,       config_class=snake_case_       )

	def 				_A			(	self       ):



				'''simple docstring'''


				self.config_tester.run_common_tests()







	def 				_A			(	self       ):



				'''simple docstring'''


				a__					=  self.model_tester.prepare_config_and_inputs_for_common()
				self.model_tester.check_decoder_model_past_large_inputs(*snake_case_       )


@require_tf
class lowerCAmelCase_       (   unittest.TestCase			):







	'''simple docstring'''




	A_       :     int									=		1000  # show more chars for failing integration tests

	@classmethod
	def 				_A			(	cls       ):



				'''simple docstring'''


				a__					=  f'Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}'
				return cls

	@cached_property
	def 				_A			(	self       ):



				'''simple docstring'''


				return AutoTokenizer.from_pretrained(self.model_name       )

	@property
	def 				_A			(	self       ):



				'''simple docstring'''


				return self.tokenizer.eos_token_id

	@cached_property
	def 				_A			(	self       ):



				'''simple docstring'''


				warnings.simplefilter("""error"""       )
				a__					=  TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name       )
				assert isinstance(snake_case_   ,       snake_case_       )
				a__					=  model.config
				self.assertListEqual(c.bad_words_ids   ,       [[c.pad_token_id]]       )
				self.assertEqual(c.max_length   ,       512       )
				self.assertEqual(c.decoder_start_token_id   ,       c.pad_token_id       )
				return model

	def 				_A			(	self   ,       **lowerCamelCase       ):



				'''simple docstring'''


				a__					=  self.translate_src_text(**snake_case_       )
				self.assertListEqual(self.expected_text   ,       snake_case_       )







	def 				_A			(	self   ,       **lowerCamelCase       ):



				'''simple docstring'''


				a__					=  self.tokenizer(self.src_text   ,       **snake_case_   ,       padding=snake_case_   ,       return_tensors="""tf"""       )
				a__					=  self.model.generate(
				    model_inputs.input_ids   ,       attention_mask=model_inputs.attention_mask   ,       num_beams=2   ,       max_length=128       )
				a__					=  self.tokenizer.batch_decode(generated_ids.numpy()   ,       skip_special_tokens=snake_case_       )
				return generated_words


@require_sentencepiece
@require_tokenizers
@require_tf
class lowerCAmelCase_       (   _UpperCAmelCase			):







	'''simple docstring'''




	A_       :     Dict									=		"""mt"""
	A_       :     str									=		"""en"""
	A_       :     List[Any]									=		["""Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."""]
	A_       :     Union[str, Any]									=		["""Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."""]

	@unittest.skip("""Skipping until #12647 is resolved."""       )
	@slow
	def 				_A			(	self       ):



				'''simple docstring'''


				self._assert_generated_batch_equal_expected()


@require_sentencepiece
@require_tokenizers
@require_tf
class lowerCAmelCase_       (   _UpperCAmelCase			):







	'''simple docstring'''




	A_       :     Optional[Any]									=		"""en"""
	A_       :     Optional[int]									=		"""zh"""
	A_       :     List[str]									=		["""My name is Wolfgang and I live in Berlin"""]
	A_       :     Any									=		["""我叫沃尔夫冈 我住在柏林"""]

	@unittest.skip("""Skipping until #12647 is resolved."""       )
	@slow
	def 				_A			(	self       ):



				'''simple docstring'''


				self._assert_generated_batch_equal_expected()


@require_sentencepiece
@require_tokenizers
@require_tf
class lowerCAmelCase_       (   _UpperCAmelCase			):







	'''simple docstring'''




	A_       :     Optional[Any]									=		"""en"""
	A_       :     Any									=		"""ROMANCE"""
	A_       :     Dict									=		[
	    """>>fr<< Don't spend so much time watching TV.""",
	    """>>pt<< Your message has been sent.""",
	    """>>es<< He's two years older than me.""",
	]
	A_       :     Tuple									=		[
	    """Ne passez pas autant de temps à regarder la télé.""",
	    """A sua mensagem foi enviada.""",
	    """Es dos años más viejo que yo.""",
	]

	@unittest.skip("""Skipping until #12647 is resolved."""       )
	@slow
	def 				_A			(	self       ):



				'''simple docstring'''


				self._assert_generated_batch_equal_expected()







	@unittest.skip("""Skipping until #12647 is resolved."""       )
	@slow
	def 				_A			(	self       ):



				'''simple docstring'''


				a__					=  TranslationPipeline(self.model   ,       self.tokenizer   ,       framework="""tf"""       )
				a__					=  pipeline(self.src_text       )
				self.assertEqual(self.expected_text   ,       [x["""translation_text"""] for x in output]       )


'''simple docstring'''

import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class 		SCREAMING_SNAKE_CASE_					(_UpperCAmelCase      ):



			"""simple docstring"""






			__magic_name__ :				Tuple   							=       ["""image_processor""", """tokenizer"""]
			__magic_name__ :				List[str]   							=       """AutoImageProcessor"""
			__magic_name__ :				List[str]   							=       """AutoTokenizer"""



			def __init__(							self				,			snake_case_=None				,			snake_case_=None				,			**snake_case_					):
									snake_case :       Any				=    None
									if "feature_extractor" in kwargs:
															warnings.warn(
															    "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
															    " instead."				,			snake_case_				,			)
															snake_case :       str				=    kwargs.pop("feature_extractor"					)

									snake_case :       Optional[int]				=    image_processor if image_processor is not None else feature_extractor
									if image_processor is None:
															raise ValueError("You have to specify an image_processor."					)
									if tokenizer is None:
															raise ValueError("You have to specify a tokenizer."					)

									super().__init__(snake_case_				,			snake_case_					)
									snake_case :       Optional[int]				=    self.image_processor



			def __call__(							self				,			snake_case_=None				,			snake_case_=None				,			snake_case_=None				,			**snake_case_					):

									if text is None and images is None:
															raise ValueError("You have to specify either text or images. Both cannot be none."					)

									if text is not None:
															snake_case :       Optional[int]				=    self.tokenizer(snake_case_				,			return_tensors=snake_case_				,			**snake_case_					)

									if images is not None:
															snake_case :       List[Any]				=    self.image_processor(snake_case_				,			return_tensors=snake_case_				,			**snake_case_					)

									if text is not None and images is not None:
															snake_case :       str				=    image_features.pixel_values
															return encoding
									elif text is not None:
															return encoding
									else:
															return BatchEncoding(data=dict(**snake_case_					)				,			tensor_type=snake_case_					)



			def   lowerCamelCase (							self				,			*snake_case_				,			**snake_case_					):
									return self.tokenizer.batch_decode(*snake_case_				,			**snake_case_					)



			def   lowerCamelCase (							self				,			*snake_case_				,			**snake_case_					):
									return self.tokenizer.decode(*snake_case_				,			**snake_case_					)



			@property
			def   lowerCamelCase (							self					):
									snake_case :       List[str]				=    self.tokenizer.model_input_names
									snake_case :       Any				=    self.image_processor.model_input_names
									return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names					)					)



			@property
			def   lowerCamelCase (							self					):
									warnings.warn(
									    "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."				,			snake_case_				,			)
									return self.image_processor_class






			@property
			def   lowerCamelCase (							self					):
									warnings.warn(
									    "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."				,			snake_case_				,			)
									return self.image_processor





