



import argparse
import collections
from pathlib import Path

import torch
from torch.serialization import default_restore_location

from transformers import BertConfig, DPRConfig, DPRContextEncoder, DPRQuestionEncoder, DPRReader


A__						:						Tuple          =      collections.namedtuple(
    '''CheckpointState''', ['''model_dict''', '''optimizer_dict''', '''scheduler_dict''', '''offset''', '''epoch''', '''encoder_params''']
)


def     UpperCamelCase(						__UpperCamelCase							:	str				):
		print(f"""Reading saved model from {model_file}"""				)
		lowerCAmelCase_    :  List[Any]             =				torch.load(__UpperCamelCase  ,map_location=lambda __UpperCamelCase  ,__UpperCamelCase				: default_restore_location(__UpperCamelCase  ,'''cpu'''				)				)
		return CheckpointState(**__UpperCamelCase				)



class        __snake_case		:




							def __init__( self  : int       ,       A_  : Path):
									lowerCAmelCase_    :  Optional[Any]             =				src_file




							def      UpperCAmelCase__    ( self  : Optional[int]):
									raise NotImplementedError







							@staticmethod
							def      UpperCAmelCase__    ( A_  : str       ,       *A_  : Dict       ,       **A_  : int):
									if comp_type.startswith('''c'''):
											return DPRContextEncoderState(*A_       ,       **A_)
									if comp_type.startswith('''q'''):
											return DPRQuestionEncoderState(*A_       ,       **A_)
									if comp_type.startswith('''r'''):
											return DPRReaderState(*A_       ,       **A_)
									else:
											raise ValueError('''Component type must be either \'ctx_encoder\', \'question_encoder\' or \'reader\'.''')





class        __snake_case		(  UpperCamelCase_		):




							def      UpperCAmelCase__    ( self  : List[str]):
									lowerCAmelCase_    :  Tuple             =				DPRContextEncoder(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased''')[0]))
									print(F"""Loading DPR biencoder from {self.src_file}""")
									lowerCAmelCase_    :  int             =				load_states_from_checkpoint(self.src_file)
									lowerCAmelCase_      ,						lowerCAmelCase_    :  Union[str, Any]             =				model.ctx_encoder, '''ctx_model.'''
									# Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
									lowerCAmelCase_    :  Dict             =				{'''bert_model.embeddings.position_ids''': model.ctx_encoder.bert_model.embeddings.position_ids}
									for key, value in saved_state.model_dict.items():
											if key.startswith(A_):
													lowerCAmelCase_    :  List[str]             =				key[len(A_) :]
													if not key.startswith('''encode_proj.'''):
															lowerCAmelCase_    :  Union[str, Any]             =				'''bert_model.''' + key
													lowerCAmelCase_    :  Optional[int]             =				value
									encoder.load_state_dict(A_)
									return model





class        __snake_case		(  UpperCamelCase_		):




							def      UpperCAmelCase__    ( self  : str):
									lowerCAmelCase_    :  Union[str, Any]             =				DPRQuestionEncoder(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased''')[0]))
									print(F"""Loading DPR biencoder from {self.src_file}""")
									lowerCAmelCase_    :  Tuple             =				load_states_from_checkpoint(self.src_file)
									lowerCAmelCase_      ,						lowerCAmelCase_    :  List[Any]             =				model.question_encoder, '''question_model.'''
									# Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
									lowerCAmelCase_    :  Tuple             =				{'''bert_model.embeddings.position_ids''': model.question_encoder.bert_model.embeddings.position_ids}
									for key, value in saved_state.model_dict.items():
											if key.startswith(A_):
													lowerCAmelCase_    :  Union[str, Any]             =				key[len(A_) :]
													if not key.startswith('''encode_proj.'''):
															lowerCAmelCase_    :  Dict             =				'''bert_model.''' + key
													lowerCAmelCase_    :  int             =				value
									encoder.load_state_dict(A_)
									return model





class        __snake_case		(  UpperCamelCase_		):




							def      UpperCAmelCase__    ( self  : str):
									lowerCAmelCase_    :  Optional[Any]             =				DPRReader(DPRConfig(**BertConfig.get_config_dict('''bert-base-uncased''')[0]))
									print(F"""Loading DPR reader from {self.src_file}""")
									lowerCAmelCase_    :  Optional[int]             =				load_states_from_checkpoint(self.src_file)
									# Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
									lowerCAmelCase_    :  Optional[int]             =				{
									    '''encoder.bert_model.embeddings.position_ids''': model.span_predictor.encoder.bert_model.embeddings.position_ids
									}
									for key, value in saved_state.model_dict.items():
											if key.startswith('''encoder.''') and not key.startswith('''encoder.encode_proj'''):
													lowerCAmelCase_    :  List[str]             =				'''encoder.bert_model.''' + key[len('''encoder.''') :]
											lowerCAmelCase_    :  List[str]             =				value
									model.span_predictor.load_state_dict(A_)
									return model





def     UpperCamelCase(						__UpperCamelCase							:	str  ,__UpperCamelCase							:	Path  ,__UpperCamelCase							:	Path				):
		lowerCAmelCase_    :  List[Any]             =				Path(__UpperCamelCase				)
		dest_dir.mkdir(exist_ok=__UpperCamelCase				)

		lowerCAmelCase_    :  Any             =				DPRState.from_type(__UpperCamelCase  ,src_file=__UpperCamelCase				)
		lowerCAmelCase_    :  List[Any]             =				dpr_state.load_dpr_model()
		model.save_pretrained(__UpperCamelCase				)
		model.from_pretrained(__UpperCamelCase				)  # sanity check


if __name__ == "__main__":
						A__						:						List[Any]          =      argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '''--type''', type=str, help='''Type of the component to convert: \'ctx_encoder\', \'question_encoder\' or \'reader\'.'''
						)
						parser.add_argument(
						    '''--src''',
						    type=str,
						    help=(
						        '''Path to the dpr checkpoint file. They can be downloaded from the official DPR repo'''
						        ''' https://github.com/facebookresearch/DPR. Note that in the official repo, both encoders are stored in the'''
						        ''' \'retriever\' checkpoints.'''
						    ),
						)
						parser.add_argument('''--dest''', type=str, default=None, help='''Path to the output PyTorch model directory.''')
						A__						:						int          =      parser.parse_args()

						A__						:						str          =      Path(args.src)
						A__						:						str          =      F'''converted-{src_file.name}''' if args.dest is None else args.dest
						A__						:						Dict          =      Path(dest_dir)
						assert src_file.exists()
						assert (
						    args.type is not None
						), "Please specify the component type of the DPR model to convert: 'ctx_encoder', 'question_encoder' or 'reader'."
						convert(args.type, src_file, dest_dir)










from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel



@require_tf
class        __snake_case		:
							_a       =				MarianConfig
							_a       =				{}
							_a       =				'''gelu'''




							def __init__( self  : Dict       ,       A_  : Optional[int]       ,       A_  : Any=1_3       ,       A_  : Any=7       ,       A_  : int=True       ,       A_  : int=False       ,       A_  : str=9_9       ,       A_  : Tuple=3_2       ,       A_  : Dict=2       ,       A_  : Union[str, Any]=4       ,       A_  : Optional[int]=3_7       ,       A_  : Any=0.1       ,       A_  : str=0.1       ,       A_  : Optional[int]=2_0       ,       A_  : Any=2       ,       A_  : Any=1       ,       A_  : Any=0       ,       ):
									lowerCAmelCase_    :  Any             =				parent
									lowerCAmelCase_    :  Dict             =				batch_size
									lowerCAmelCase_    :  Any             =				seq_length
									lowerCAmelCase_    :  Union[str, Any]             =				is_training
									lowerCAmelCase_    :  Any             =				use_labels
									lowerCAmelCase_    :  Tuple             =				vocab_size
									lowerCAmelCase_    :  Optional[int]             =				hidden_size
									lowerCAmelCase_    :  Optional[int]             =				num_hidden_layers
									lowerCAmelCase_    :  Union[str, Any]             =				num_attention_heads
									lowerCAmelCase_    :  List[str]             =				intermediate_size

									lowerCAmelCase_    :  Union[str, Any]             =				hidden_dropout_prob
									lowerCAmelCase_    :  Union[str, Any]             =				attention_probs_dropout_prob
									lowerCAmelCase_    :  Optional[Any]             =				max_position_embeddings
									lowerCAmelCase_    :  List[str]             =				eos_token_id
									lowerCAmelCase_    :  Optional[Any]             =				pad_token_id
									lowerCAmelCase_    :  Dict             =				bos_token_id




							def      UpperCAmelCase__    ( self  : Tuple):
									lowerCAmelCase_    :  str             =				ids_tensor([self.batch_size, self.seq_length - 1]       ,       self.vocab_size)
									lowerCAmelCase_    :  Optional[int]             =				tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size)       ,       1)
									lowerCAmelCase_    :  List[str]             =				tf.concat([input_ids, eos_tensor]       ,       axis=1)

									lowerCAmelCase_    :  Optional[Any]             =				ids_tensor([self.batch_size, self.seq_length]       ,       self.vocab_size)

									lowerCAmelCase_    :  Optional[int]             =				self.config_cls(
									    vocab_size=self.vocab_size       ,       d_model=self.hidden_size       ,       encoder_layers=self.num_hidden_layers       ,       decoder_layers=self.num_hidden_layers       ,       encoder_attention_heads=self.num_attention_heads       ,       decoder_attention_heads=self.num_attention_heads       ,       encoder_ffn_dim=self.intermediate_size       ,       decoder_ffn_dim=self.intermediate_size       ,       dropout=self.hidden_dropout_prob       ,       attention_dropout=self.attention_probs_dropout_prob       ,       max_position_embeddings=self.max_position_embeddings       ,       eos_token_ids=[2]       ,       bos_token_id=self.bos_token_id       ,       pad_token_id=self.pad_token_id       ,       decoder_start_token_id=self.pad_token_id       ,       **self.config_updates       ,       )
									lowerCAmelCase_    :  int             =				prepare_marian_inputs_dict(A_       ,       A_       ,       A_)
									return config, inputs_dict







							def      UpperCAmelCase__    ( self  : int       ,       A_  : int       ,       A_  : str):
									lowerCAmelCase_    :  Any             =				TFMarianModel(config=A_).get_decoder()
									lowerCAmelCase_    :  List[Any]             =				inputs_dict['''input_ids''']

									lowerCAmelCase_    :  str             =				input_ids[:1, :]
									lowerCAmelCase_    :  List[Any]             =				inputs_dict['''attention_mask'''][:1, :]
									lowerCAmelCase_    :  Dict             =				inputs_dict['''head_mask''']
									lowerCAmelCase_    :  Optional[int]             =				1

									# first forward pass
									lowerCAmelCase_    :  str             =				model(A_       ,       attention_mask=A_       ,       head_mask=A_       ,       use_cache=A_)

									lowerCAmelCase_      ,						lowerCAmelCase_    :  Any             =				outputs.to_tuple()

									# create hypothetical next token and extent to next_input_ids
									lowerCAmelCase_    :  int             =				ids_tensor((self.batch_size, 3)       ,       config.vocab_size)
									lowerCAmelCase_    :  str             =				tf.cast(ids_tensor((self.batch_size, 3)       ,       2)       ,       tf.inta)

									# append to next input_ids and
									lowerCAmelCase_    :  Optional[int]             =				tf.concat([input_ids, next_tokens]       ,       axis=-1)
									lowerCAmelCase_    :  int             =				tf.concat([attention_mask, next_attn_mask]       ,       axis=-1)

									lowerCAmelCase_    :  str             =				model(A_       ,       attention_mask=A_)[0]
									lowerCAmelCase_    :  List[str]             =				model(A_       ,       attention_mask=A_       ,       past_key_values=A_)[0]

									self.parent.assertEqual(next_tokens.shape[1]       ,       output_from_past.shape[1])

									# select random slice
									lowerCAmelCase_    :  Optional[Any]             =				int(ids_tensor((1,)       ,       output_from_past.shape[-1]))
									lowerCAmelCase_    :  Tuple             =				output_from_no_past[:, -3:, random_slice_idx]
									lowerCAmelCase_    :  Optional[Any]             =				output_from_past[:, :, random_slice_idx]

									# test that outputs are equal for slice
									tf.debugging.assert_near(A_       ,       A_       ,       rtol=1e-3)





def     UpperCamelCase(						__UpperCamelCase							:	List[str]  ,__UpperCamelCase							:	int  ,__UpperCamelCase							:	Dict  ,__UpperCamelCase							:	List[str]=None  ,__UpperCamelCase							:	Union[str, Any]=None  ,__UpperCamelCase							:	List[Any]=None  ,__UpperCamelCase							:	str=None  ,__UpperCamelCase							:	Optional[int]=None  ,):
		if attention_mask is None:
				lowerCAmelCase_    :  Optional[int]             =				tf.cast(tf.math.not_equal(__UpperCamelCase  ,config.pad_token_id				)  ,tf.inta				)
		if decoder_attention_mask is None:
				lowerCAmelCase_    :  List[str]             =				tf.concat(
				    [
				        tf.ones(decoder_input_ids[:, :1].shape  ,dtype=tf.inta				),
				        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]  ,config.pad_token_id				)  ,tf.inta				),
				    ]  ,axis=-1  ,)
		if head_mask is None:
				lowerCAmelCase_    :  Union[str, Any]             =				tf.ones((config.encoder_layers, config.encoder_attention_heads)				)
		if decoder_head_mask is None:
				lowerCAmelCase_    :  List[str]             =				tf.ones((config.decoder_layers, config.decoder_attention_heads)				)
		if cross_attn_head_mask is None:
				lowerCAmelCase_    :  Union[str, Any]             =				tf.ones((config.decoder_layers, config.decoder_attention_heads)				)
		return {
		    "input_ids": input_ids,
		    "decoder_input_ids": decoder_input_ids,
		    "attention_mask": attention_mask,
		    "decoder_attention_mask": decoder_attention_mask,
		    "head_mask": head_mask,
		    "decoder_head_mask": decoder_head_mask,
		    "cross_attn_head_mask": cross_attn_head_mask,
		}



@require_tf
class        __snake_case		(  UpperCamelCase_		,UpperCamelCase_		,unittest.TestCase		):
							_a       =				(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
							_a       =				(TFMarianMTModel,) if is_tf_available() else ()
							_a       =				(
							    {
							        '''conversational''': TFMarianMTModel,
							        '''feature-extraction''': TFMarianModel,
							        '''summarization''': TFMarianMTModel,
							        '''text2text-generation''': TFMarianMTModel,
							        '''translation''': TFMarianMTModel,
							    }
							    if is_tf_available()
							    else {}
							)
							_a       =				True
							_a       =				False
							_a       =				False




							def      UpperCAmelCase__    ( self  : Union[str, Any]):
									lowerCAmelCase_    :  Optional[Any]             =				TFMarianModelTester(self)
									lowerCAmelCase_    :  List[str]             =				ConfigTester(self       ,       config_class=A_)




							def      UpperCAmelCase__    ( self  : List[Any]):
									self.config_tester.run_common_tests()







							def      UpperCAmelCase__    ( self  : List[Any]):
									lowerCAmelCase_    :  List[Any]             =				self.model_tester.prepare_config_and_inputs_for_common()
									self.model_tester.check_decoder_model_past_large_inputs(*A_)





@require_tf
class        __snake_case		(  unittest.TestCase		):
							_a       =				1_000  # show more chars for failing integration tests




							@classmethod
							def      UpperCAmelCase__    ( cls  : Optional[Any]):
									lowerCAmelCase_    :  Union[str, Any]             =				F"""Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"""
									return cls




							@cached_property
							def      UpperCAmelCase__    ( self  : List[str]):
									return AutoTokenizer.from_pretrained(self.model_name)




							@property
							def      UpperCAmelCase__    ( self  : Dict):
									return self.tokenizer.eos_token_id




							@cached_property
							def      UpperCAmelCase__    ( self  : int):
									warnings.simplefilter('''error''')
									lowerCAmelCase_    :  TFMarianMTModel             =				TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name)
									assert isinstance(A_       ,       A_)
									lowerCAmelCase_    :  int             =				model.config
									self.assertListEqual(c.bad_words_ids       ,       [[c.pad_token_id]])
									self.assertEqual(c.max_length       ,       5_1_2)
									self.assertEqual(c.decoder_start_token_id       ,       c.pad_token_id)
									return model




							def      UpperCAmelCase__    ( self  : Dict       ,       **A_  : Tuple):
									lowerCAmelCase_    :  Optional[int]             =				self.translate_src_text(**A_)
									self.assertListEqual(self.expected_text       ,       A_)







							def      UpperCAmelCase__    ( self  : Any       ,       **A_  : List[str]):
									lowerCAmelCase_    :  List[Any]             =				self.tokenizer(self.src_text       ,       **A_       ,       padding=A_       ,       return_tensors='''tf''')
									lowerCAmelCase_    :  List[str]             =				self.model.generate(
									    model_inputs.input_ids       ,       attention_mask=model_inputs.attention_mask       ,       num_beams=2       ,       max_length=1_2_8)
									lowerCAmelCase_    :  Any             =				self.tokenizer.batch_decode(generated_ids.numpy()       ,       skip_special_tokens=A_)
									return generated_words





@require_sentencepiece
@require_tokenizers
@require_tf
class        __snake_case		(  UpperCamelCase_		):
							_a       =				'''mt'''
							_a       =				'''en'''
							_a       =				['''Billi messu b\'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem.''']
							_a       =				['''Touching gently, Jesus healed a man who was affected by the sad disease of leprosy.''']




							@unittest.skip('''Skipping until #12647 is resolved.''')
							@slow
							def      UpperCAmelCase__    ( self  : Any):
									self._assert_generated_batch_equal_expected()





@require_sentencepiece
@require_tokenizers
@require_tf
class        __snake_case		(  UpperCamelCase_		):
							_a       =				'''en'''
							_a       =				'''zh'''
							_a       =				['''My name is Wolfgang and I live in Berlin''']
							_a       =				['''我叫沃尔夫冈 我住在柏林''']




							@unittest.skip('''Skipping until #12647 is resolved.''')
							@slow
							def      UpperCAmelCase__    ( self  : str):
									self._assert_generated_batch_equal_expected()





@require_sentencepiece
@require_tokenizers
@require_tf
class        __snake_case		(  UpperCamelCase_		):
							_a       =				'''en'''
							_a       =				'''ROMANCE'''
							_a       =				[
							    '''>>fr<< Don\'t spend so much time watching TV.''',
							    '''>>pt<< Your message has been sent.''',
							    '''>>es<< He\'s two years older than me.''',
							]
							_a       =				[
							    '''Ne passez pas autant de temps à regarder la télé.''',
							    '''A sua mensagem foi enviada.''',
							    '''Es dos años más viejo que yo.''',
							]




							@unittest.skip('''Skipping until #12647 is resolved.''')
							@slow
							def      UpperCAmelCase__    ( self  : Union[str, Any]):
									self._assert_generated_batch_equal_expected()







							@unittest.skip('''Skipping until #12647 is resolved.''')
							@slow
							def      UpperCAmelCase__    ( self  : Optional[int]):
									lowerCAmelCase_    :  Dict             =				TranslationPipeline(self.model       ,       self.tokenizer       ,       framework='''tf''')
									lowerCAmelCase_    :  Union[str, Any]             =				pipeline(self.src_text)
									self.assertEqual(self.expected_text       ,       [x['''translation_text'''] for x in output])





