




import argparse
import json
import re
import string
import sys
from collections import Counter






def       snake_case      (							A    )							->							Any:




	'''simple docstring'''


	def remove_articles(A    ):
		return re.sub(R'\b(a|an|the)\b'		,     ' '		,     __lowerCamelCase    )

	def white_space_fix(A    ):
		return " ".join(text.split()    )

	def remove_punc(A    ):
		lowerCamelCase_							:	List[Any]							     =							set(string.punctuation    )
		return "".join(ch for ch in text if ch not in exclude    )

	def lower(A    ):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(__lowerCamelCase    )    )    )    )






def       snake_case      (							A		,     A    )							->							Optional[Any]:




	'''simple docstring'''


	lowerCamelCase_							:	Optional[Any]							     =							normalize_answer(__lowerCamelCase    ).split()
	lowerCamelCase_							:	List[str]							     =							normalize_answer(__lowerCamelCase    ).split()
	lowerCamelCase_							:	int							     =							Counter(__lowerCamelCase    ) & Counter(__lowerCamelCase    )
	lowerCamelCase_							:	List[str]							     =							sum(common.values()    )
	if num_same == 0:
		return 0
	lowerCamelCase_							:	List[Any]							     =							1.0 * num_same / len(__lowerCamelCase    )
	lowerCamelCase_							:	int							     =							1.0 * num_same / len(__lowerCamelCase    )
	lowerCamelCase_							:	Dict							     =							(2 * precision * recall) / (precision + recall)
	return fa






def       snake_case      (							A		,     A    )							->							Optional[Any]:




	'''simple docstring'''


	return normalize_answer(__lowerCamelCase    ) == normalize_answer(__lowerCamelCase    )






def       snake_case      (							A		,     A		,     A    )							->							str:




	'''simple docstring'''


	lowerCamelCase_							:	Optional[Any]							     =							[]
	for ground_truth in ground_truths:
		lowerCamelCase_							:	int							     =							metric_fn(__lowerCamelCase		,     __lowerCamelCase    )
		scores_for_ground_truths.append(__lowerCamelCase    )
	return max(__lowerCamelCase    )






def       snake_case      (							A		,     A    )							->							Tuple:




	'''simple docstring'''


	lowerCamelCase_							:	Tuple							     =							0
	for article in dataset:
		for paragraph in article["paragraphs"]:
			for qa in paragraph["qas"]:
				total += 1
				if qa["id"] not in predictions:
					lowerCamelCase_							:	Dict							     =							'Unanswered question ' + qa['id'] + ' will receive score 0.'
					print(__lowerCamelCase		,     file=sys.stderr    )
					continue
				lowerCamelCase_							:	Any							     =							[x['text'] for x in qa['answers']]
				lowerCamelCase_							:	Any							     =							predictions[qa['id']]
				exact_match += metric_max_over_ground_truths(__lowerCamelCase		,     __lowerCamelCase		,     __lowerCamelCase    )
				fa += metric_max_over_ground_truths(__lowerCamelCase		,     __lowerCamelCase		,     __lowerCamelCase    )

	lowerCamelCase_							:	Tuple							     =							100.0 * exact_match / total
	lowerCamelCase_							:	Any							     =							100.0 * fa / total

	return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
	UpperCamelCase__  =		'1.1'
	UpperCamelCase__  =		argparse.ArgumentParser(description='Evaluation for SQuAD ' + expected_version)
	parser.add_argument('dataset_file', help='Dataset file')
	parser.add_argument('prediction_file', help='Prediction File')
	UpperCamelCase__  =		parser.parse_args()
	with open(args.dataset_file) as dataset_file:
		UpperCamelCase__  =		json.load(dataset_file)
		if dataset_json["version"] != expected_version:
			print(
			    'Evaluation expects v-' + expected_version + ', but got dataset with v-' + dataset_json['version'],
			    file=sys.stderr,
			)
		UpperCamelCase__  =		dataset_json['data']
	with open(args.prediction_file) as prediction_file:
		UpperCamelCase__  =		json.load(prediction_file)
	print(json.dumps(evaluate(dataset, predictions)))






from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel


@require_tf
class        __SCREAMING_SNAKE_CASE							:
					_lowerCamelCase							=				MarianConfig
					_lowerCamelCase							=				{}
					_lowerCamelCase							=				"""gelu"""

					def __init__(       self				, __lowercase				, __lowercase=13				, __lowercase=7				, __lowercase=True				, __lowercase=False				, __lowercase=99				, __lowercase=32				, __lowercase=2				, __lowercase=4				, __lowercase=37				, __lowercase=0.1				, __lowercase=0.1				, __lowercase=20				, __lowercase=2				, __lowercase=1				, __lowercase=0				, ):







						'''simple docstring'''




						lowerCamelCase_							:	Optional[Any]							     =							parent
						lowerCamelCase_							:	List[str]							     =							batch_size
						lowerCamelCase_							:	List[str]							     =							seq_length
						lowerCamelCase_							:	Optional[int]							     =							is_training
						lowerCamelCase_							:	List[str]							     =							use_labels
						lowerCamelCase_							:	Tuple							     =							vocab_size
						lowerCamelCase_							:	Dict							     =							hidden_size
						lowerCamelCase_							:	List[str]							     =							num_hidden_layers
						lowerCamelCase_							:	List[str]							     =							num_attention_heads
						lowerCamelCase_							:	Tuple							     =							intermediate_size

						lowerCamelCase_							:	Tuple							     =							hidden_dropout_prob
						lowerCamelCase_							:	Dict							     =							attention_probs_dropout_prob
						lowerCamelCase_							:	List[str]							     =							max_position_embeddings
						lowerCamelCase_							:	Optional[int]							     =							eos_token_id
						lowerCamelCase_							:	List[Any]							     =							pad_token_id
						lowerCamelCase_							:	Optional[int]							     =							bos_token_id

					def 			lowercase_							(       self      ):







						'''simple docstring'''




						lowerCamelCase_							:	Union[str, Any]							     =							ids_tensor([self.batch_size, self.seq_length - 1]				, self.vocab_size      )
						lowerCamelCase_							:	Tuple							     =							tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size      )				, 1      )
						lowerCamelCase_							:	Optional[int]							     =							tf.concat([input_ids, eos_tensor]				, axis=1      )

						lowerCamelCase_							:	List[Any]							     =							ids_tensor([self.batch_size, self.seq_length]				, self.vocab_size      )

						lowerCamelCase_							:	Optional[Any]							     =							self.config_cls(
						    vocab_size=self.vocab_size				, d_model=self.hidden_size				, encoder_layers=self.num_hidden_layers				, decoder_layers=self.num_hidden_layers				, encoder_attention_heads=self.num_attention_heads				, decoder_attention_heads=self.num_attention_heads				, encoder_ffn_dim=self.intermediate_size				, decoder_ffn_dim=self.intermediate_size				, dropout=self.hidden_dropout_prob				, attention_dropout=self.attention_probs_dropout_prob				, max_position_embeddings=self.max_position_embeddings				, eos_token_ids=[2]				, bos_token_id=self.bos_token_id				, pad_token_id=self.pad_token_id				, decoder_start_token_id=self.pad_token_id				, **self.config_updates				, )
						lowerCamelCase_							:	int							     =							prepare_marian_inputs_dict(__lowercase				, __lowercase				, __lowercase      )
						return config, inputs_dict





					def 			lowercase_							(       self				, __lowercase				, __lowercase      ):







						'''simple docstring'''




						lowerCamelCase_							:	Union[str, Any]							     =							TFMarianModel(config=__lowercase      ).get_decoder()
						lowerCamelCase_							:	Tuple							     =							inputs_dict['input_ids']

						lowerCamelCase_							:	Union[str, Any]							     =							input_ids[:1, :]
						lowerCamelCase_							:	List[str]							     =							inputs_dict['attention_mask'][:1, :]
						lowerCamelCase_							:	List[str]							     =							inputs_dict['head_mask']
						lowerCamelCase_							:	List[str]							     =							1

						# first forward pass
						lowerCamelCase_							:	List[str]							     =							model(__lowercase				, attention_mask=__lowercase				, head_mask=__lowercase				, use_cache=__lowercase      )

						lowerCamelCase_     ,    lowerCamelCase_							:	Any							     =							outputs.to_tuple()

						# create hypothetical next token and extent to next_input_ids
						lowerCamelCase_							:	Any							     =							ids_tensor((self.batch_size, 3)				, config.vocab_size      )
						lowerCamelCase_							:	Optional[Any]							     =							tf.cast(ids_tensor((self.batch_size, 3)				, 2      )				, tf.inta      )

						# append to next input_ids and
						lowerCamelCase_							:	str							     =							tf.concat([input_ids, next_tokens]				, axis=-1      )
						lowerCamelCase_							:	List[str]							     =							tf.concat([attention_mask, next_attn_mask]				, axis=-1      )

						lowerCamelCase_							:	str							     =							model(__lowercase				, attention_mask=__lowercase      )[0]
						lowerCamelCase_							:	Optional[Any]							     =							model(__lowercase				, attention_mask=__lowercase				, past_key_values=__lowercase      )[0]

						self.parent.assertEqual(next_tokens.shape[1]				, output_from_past.shape[1]      )

						# select random slice
						lowerCamelCase_							:	Any							     =							int(ids_tensor((1,)				, output_from_past.shape[-1]      )      )
						lowerCamelCase_							:	Union[str, Any]							     =							output_from_no_past[:, -3:, random_slice_idx]
						lowerCamelCase_							:	int							     =							output_from_past[:, :, random_slice_idx]

						# test that outputs are equal for slice
						tf.debugging.assert_near(__lowercase				, __lowercase				, rtol=1E-3      )






def       snake_case      (							A		,     A		,     A		,     A=None		,     A=None		,     A=None		,     A=None		,     A=None		,     )							->							Dict:




	'''simple docstring'''


	if attention_mask is None:
		lowerCamelCase_							:	Tuple							     =							tf.cast(tf.math.not_equal(A		,     config.pad_token_id    )		,     tf.inta    )
	if decoder_attention_mask is None:
		lowerCamelCase_							:	Union[str, Any]							     =							tf.concat(
		    [
		        tf.ones(decoder_input_ids[:, :1].shape		,     dtype=tf.inta    ),
		        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]		,     config.pad_token_id    )		,     tf.inta    ),
		    ]		,     axis=-1		,     )
	if head_mask is None:
		lowerCamelCase_							:	Optional[Any]							     =							tf.ones((config.encoder_layers, config.encoder_attention_heads)    )
	if decoder_head_mask is None:
		lowerCamelCase_							:	Tuple							     =							tf.ones((config.decoder_layers, config.decoder_attention_heads)    )
	if cross_attn_head_mask is None:
		lowerCamelCase_							:	Dict							     =							tf.ones((config.decoder_layers, config.decoder_attention_heads)    )
	return {
	    "input_ids": input_ids,
	    "decoder_input_ids": decoder_input_ids,
	    "attention_mask": attention_mask,
	    "decoder_attention_mask": decoder_attention_mask,
	    "head_mask": head_mask,
	    "decoder_head_mask": decoder_head_mask,
	    "cross_attn_head_mask": cross_attn_head_mask,
	}


@require_tf
class        __SCREAMING_SNAKE_CASE							(	lowerCamelCase     ,  lowerCamelCase     ,  unittest.TestCase						):
					_lowerCamelCase							=				(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
					_lowerCamelCase							=				(TFMarianMTModel,) if is_tf_available() else ()
					_lowerCamelCase							=				(
					    {
					        """conversational""": TFMarianMTModel,
					        """feature-extraction""": TFMarianModel,
					        """summarization""": TFMarianMTModel,
					        """text2text-generation""": TFMarianMTModel,
					        """translation""": TFMarianMTModel,
					    }
					    if is_tf_available()
					    else {}
					)
					_lowerCamelCase							=				True
					_lowerCamelCase							=				False
					_lowerCamelCase							=				False

					def 			lowercase_							(       self      ):







						'''simple docstring'''




						lowerCamelCase_							:	Union[str, Any]							     =							TFMarianModelTester(self      )
						lowerCamelCase_							:	str							     =							ConfigTester(self				, config_class=__lowercase      )

					def 			lowercase_							(       self      ):







						'''simple docstring'''




						self.config_tester.run_common_tests()





					def 			lowercase_							(       self      ):







						'''simple docstring'''




						lowerCamelCase_							:	Optional[Any]							     =							self.model_tester.prepare_config_and_inputs_for_common()
						self.model_tester.check_decoder_model_past_large_inputs(*__lowercase      )


@require_tf
class        __SCREAMING_SNAKE_CASE							(	unittest.TestCase						):
					_lowerCamelCase							=				10_00  # show more chars for failing integration tests

					@classmethod
					def 			lowercase_							(       cls      ):







						'''simple docstring'''




						lowerCamelCase_							:	Union[str, Any]							     =							f"Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"
						return cls

					@cached_property
					def 			lowercase_							(       self      ):







						'''simple docstring'''




						return AutoTokenizer.from_pretrained(self.model_name      )

					@property
					def 			lowercase_							(       self      ):







						'''simple docstring'''




						return self.tokenizer.eos_token_id

					@cached_property
					def 			lowercase_							(       self      ):







						'''simple docstring'''




						warnings.simplefilter('error'      )
						lowerCamelCase_							:	TFMarianMTModel							     =							TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name      )
						assert isinstance(__lowercase				, __lowercase      )
						lowerCamelCase_							:	Optional[Any]							     =							model.config
						self.assertListEqual(c.bad_words_ids				, [[c.pad_token_id]]      )
						self.assertEqual(c.max_length				, 512      )
						self.assertEqual(c.decoder_start_token_id				, c.pad_token_id      )
						return model

					def 			lowercase_							(       self				, **__lowercase      ):







						'''simple docstring'''




						lowerCamelCase_							:	List[str]							     =							self.translate_src_text(**__lowercase      )
						self.assertListEqual(self.expected_text				, __lowercase      )





					def 			lowercase_							(       self				, **__lowercase      ):







						'''simple docstring'''




						lowerCamelCase_							:	str							     =							self.tokenizer(self.src_text				, **__lowercase				, padding=__lowercase				, return_tensors='tf'      )
						lowerCamelCase_							:	Optional[int]							     =							self.model.generate(
						    model_inputs.input_ids				, attention_mask=model_inputs.attention_mask				, num_beams=2				, max_length=128      )
						lowerCamelCase_							:	Optional[int]							     =							self.tokenizer.batch_decode(generated_ids.numpy()				, skip_special_tokens=__lowercase      )
						return generated_words


@require_sentencepiece
@require_tokenizers
@require_tf
class        __SCREAMING_SNAKE_CASE							(	lowerCamelCase						):
					_lowerCamelCase							=				"""mt"""
					_lowerCamelCase							=				"""en"""
					_lowerCamelCase							=				["""Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."""]
					_lowerCamelCase							=				["""Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."""]

					@unittest.skip('Skipping until #12647 is resolved.'      )
					@slow
					def 			lowercase_							(       self      ):







						'''simple docstring'''




						self._assert_generated_batch_equal_expected()


@require_sentencepiece
@require_tokenizers
@require_tf
class        __SCREAMING_SNAKE_CASE							(	lowerCamelCase						):
					_lowerCamelCase							=				"""en"""
					_lowerCamelCase							=				"""zh"""
					_lowerCamelCase							=				["""My name is Wolfgang and I live in Berlin"""]
					_lowerCamelCase							=				["""我叫沃尔夫冈 我住在柏林"""]

					@unittest.skip('Skipping until #12647 is resolved.'      )
					@slow
					def 			lowercase_							(       self      ):







						'''simple docstring'''




						self._assert_generated_batch_equal_expected()


@require_sentencepiece
@require_tokenizers
@require_tf
class        __SCREAMING_SNAKE_CASE							(	lowerCamelCase						):
					_lowerCamelCase							=				"""en"""
					_lowerCamelCase							=				"""ROMANCE"""
					_lowerCamelCase							=				[
					    """>>fr<< Don't spend so much time watching TV.""",
					    """>>pt<< Your message has been sent.""",
					    """>>es<< He's two years older than me.""",
					]
					_lowerCamelCase							=				[
					    """Ne passez pas autant de temps à regarder la télé.""",
					    """A sua mensagem foi enviada.""",
					    """Es dos años más viejo que yo.""",
					]

					@unittest.skip('Skipping until #12647 is resolved.'      )
					@slow
					def 			lowercase_							(       self      ):







						'''simple docstring'''




						self._assert_generated_batch_equal_expected()





					@unittest.skip('Skipping until #12647 is resolved.'      )
					@slow
					def 			lowercase_							(       self      ):







						'''simple docstring'''




						lowerCamelCase_							:	Optional[int]							     =							TranslationPipeline(self.model				, self.tokenizer				, framework='tf'      )
						lowerCamelCase_							:	Optional[int]							     =							pipeline(self.src_text      )
						self.assertEqual(self.expected_text				, [x['translation_text'] for x in output]      )
