import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
 import jax
 import jax.numpy as jnp

 from transformers.modeling_flax_pytorch_utils import (
     convert_pytorch_state_dict_to_flax,
     load_flax_weights_in_pytorch_model,
 )
 from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
 import torch




class  __lowerCAmelCase	:



 '''simple docstring'''

 def __init__(      self							:		int							,      lowerCamelCase							:		Union[str, Any]							,      lowerCamelCase							:		Dict=1_4							,      lowerCamelCase							:		Dict=7							,      lowerCamelCase							:		str=True							,      lowerCamelCase							:		List[Any]=True							,      lowerCamelCase							:		Any=False							,      lowerCamelCase							:		int=True							,      lowerCamelCase							:		List[str]=9_9							,      lowerCamelCase							:		Dict=3_2							,      lowerCamelCase							:		Any=5							,      lowerCamelCase							:		List[Any]=4							,      lowerCamelCase							:		Tuple=3_7							,      lowerCamelCase							:		Tuple="gelu"							,      lowerCamelCase							:		int=0.1							,      lowerCamelCase							:		List[str]=0.1							,      lowerCamelCase							:		int=5_1_2							,      lowerCamelCase							:		Union[str, Any]=0.0_2							,      )			->				Optional[int]:
        _UpperCamelCase  :							Any	    =	parent
        _UpperCamelCase  :							List[Any]	    =	batch_size
        _UpperCamelCase  :							List[str]	    =	seq_length
        _UpperCamelCase  :							Any	    =	is_training
        _UpperCamelCase  :							Optional[int]	    =	use_input_mask
        _UpperCamelCase  :							int	    =	use_token_type_ids
        _UpperCamelCase  :							Any	    =	use_labels
        _UpperCamelCase  :							str	    =	vocab_size
        _UpperCamelCase  :							Any	    =	hidden_size
        _UpperCamelCase  :							Any	    =	num_hidden_layers
        _UpperCamelCase  :							int	    =	num_attention_heads
        _UpperCamelCase  :							Dict	    =	intermediate_size
        _UpperCamelCase  :							Any	    =	hidden_act
        _UpperCamelCase  :							Optional[int]	    =	hidden_dropout_prob
        _UpperCamelCase  :							Any	    =	attention_probs_dropout_prob
        _UpperCamelCase  :							Tuple	    =	max_position_embeddings
        _UpperCamelCase  :							Tuple	    =	initializer_range
        _UpperCamelCase  :							Union[str, Any]	    =	None
        _UpperCamelCase  :							Dict	    =	vocab_size - 1
        _UpperCamelCase  :							Optional[int]	    =	vocab_size - 1
        _UpperCamelCase  :							Optional[Any]	    =	vocab_size - 1

 def        __snake_case       (      self							:		List[Any]	)			->				Any:
        _UpperCamelCase  :							Dict	    =	ids_tensor([self.batch_size, self.seq_length]							,      self.vocab_size	)

        _UpperCamelCase  :							Optional[int]	    =	None
        if self.use_input_mask:
               _UpperCamelCase  :							Union[str, Any]	    =	random_attention_mask([self.batch_size, self.seq_length]	)

        _UpperCamelCase  :							str	    =	GPTaConfig(
            vocab_size=self.vocab_size							,      n_embd=self.hidden_size							,      n_layer=self.num_hidden_layers							,      n_head=self.num_attention_heads							,      n_positions=self.max_position_embeddings							,      use_cache=lowerCamelCase							,      bos_token_id=self.bos_token_id							,      eos_token_id=self.eos_token_id							,      pad_token_id=self.pad_token_id							,      )

        return (config, input_ids, input_mask)

 def        __snake_case       (      self							:		int	)			->				Union[str, Any]:
        _UpperCamelCase  :							Any	    =	self.prepare_config_and_inputs()
        _UpperCamelCase    ,   _UpperCamelCase    ,   _UpperCamelCase  :							Optional[int]	    =	config_and_inputs
        _UpperCamelCase  :							Optional[int]	    =	{'input_ids': input_ids, 'attention_mask': attention_mask}
        return config, inputs_dict

 def        __snake_case       (      self							:		Optional[int]	)			->				List[str]:
        _UpperCamelCase    ,   _UpperCamelCase    ,   _UpperCamelCase  :							int	    =	self.prepare_config_and_inputs()

        _UpperCamelCase  :							List[str]	    =	floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
        _UpperCamelCase  :							str	    =	ids_tensor([self.batch_size, self.seq_length]							,      vocab_size=2	)

        return (
            config,
            input_ids,
            attention_mask,
            encoder_hidden_states,
            encoder_attention_mask,
        )

 def        __snake_case       (      self							:		str							,      lowerCamelCase							:		Union[str, Any]							,      lowerCamelCase							:		str							,      lowerCamelCase							:		List[Any]							,      lowerCamelCase							:		Optional[int]	)			->				Tuple:
        _UpperCamelCase  :							str	    =	2_0
        _UpperCamelCase  :							Any	    =	model_class_name(lowerCamelCase	)

        _UpperCamelCase  :							List[str]	    =	model.init_cache(input_ids.shape[0]							,      lowerCamelCase	)
        _UpperCamelCase  :							List[Any]	    =	jnp.ones((input_ids.shape[0], max_decoder_length)							,      dtype='i4'	)

        _UpperCamelCase  :							Optional[Any]	    =	jnp.broadcast_to(
            jnp.arange(input_ids.shape[-1] - 1	)[None, :]							,      (input_ids.shape[0], input_ids.shape[-1] - 1)	)
        _UpperCamelCase  :							Any	    =	model(
            input_ids[:, :-1]							,      attention_mask=lowerCamelCase							,      past_key_values=lowerCamelCase							,      position_ids=lowerCamelCase							,      )

        _UpperCamelCase  :							str	    =	jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]							,      dtype='i4'	)
        _UpperCamelCase  :							List[str]	    =	model(
            input_ids[:, -1:]							,      attention_mask=lowerCamelCase							,      past_key_values=outputs_cache.past_key_values							,      position_ids=lowerCamelCase							,      )

        _UpperCamelCase  :							Dict	    =	model(lowerCamelCase	)

        _UpperCamelCase  :							int	    =	np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])	)	)
        self.parent.assertTrue(diff < 1e-3							,      msg=f"Max diff is {diff}"	)

 def        __snake_case       (      self							:		Optional[int]							,      lowerCamelCase							:		Optional[Any]							,      lowerCamelCase							:		List[Any]							,      lowerCamelCase							:		int							,      lowerCamelCase							:		List[Any]	)			->				List[Any]:
        _UpperCamelCase  :							Dict	    =	2_0
        _UpperCamelCase  :							Optional[Any]	    =	model_class_name(lowerCamelCase	)

        _UpperCamelCase  :							Optional[Any]	    =	jnp.concatenate(
            [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])	)]							,      axis=-1							,      )

        _UpperCamelCase  :							Optional[Any]	    =	model.init_cache(input_ids.shape[0]							,      lowerCamelCase	)
        _UpperCamelCase  :							Tuple	    =	jnp.broadcast_to(
            jnp.arange(input_ids.shape[-1] - 1	)[None, :]							,      (input_ids.shape[0], input_ids.shape[-1] - 1)	)

        _UpperCamelCase  :							str	    =	model(
            input_ids[:, :-1]							,      attention_mask=lowerCamelCase							,      past_key_values=lowerCamelCase							,      position_ids=lowerCamelCase							,      )
        _UpperCamelCase  :							Optional[int]	    =	jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]							,      dtype='i4'	)
        _UpperCamelCase  :							Tuple	    =	model(
            input_ids[:, -1:]							,      past_key_values=outputs_cache.past_key_values							,      attention_mask=lowerCamelCase							,      position_ids=lowerCamelCase							,      )

        _UpperCamelCase  :							Tuple	    =	model(lowerCamelCase							,      attention_mask=lowerCamelCase	)

        _UpperCamelCase  :							List[Any]	    =	np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])	)	)
        self.parent.assertTrue(diff < 1e-3							,      msg=f"Max diff is {diff}"	)







@require_flax
class  __lowerCAmelCase	(							a_,						a_,						unittest.TestCase    ):



 '''simple docstring'''
 _lowerCamelCase    :						Tuple					     =			(FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
 _lowerCamelCase    :						List[str]					     =			(FlaxGPTaLMHeadModel,) if is_flax_available() else ()

 def        __snake_case       (      self							:		Dict	)			->				Union[str, Any]:
        _UpperCamelCase  :							List[str]	    =	FlaxGPTaModelTester(self	)

 def        __snake_case       (      self							:		Optional[Any]	)			->				Tuple:
        for model_class_name in self.all_model_classes:
               _UpperCamelCase    ,   _UpperCamelCase    ,   _UpperCamelCase  :							Union[str, Any]	    =	self.model_tester.prepare_config_and_inputs()
               self.model_tester.check_use_cache_forward(lowerCamelCase							,      lowerCamelCase							,      lowerCamelCase							,      lowerCamelCase	)

 def        __snake_case       (      self							:		Union[str, Any]	)			->				List[Any]:
        for model_class_name in self.all_model_classes:
               _UpperCamelCase    ,   _UpperCamelCase    ,   _UpperCamelCase  :							Union[str, Any]	    =	self.model_tester.prepare_config_and_inputs()
               self.model_tester.check_use_cache_forward_with_attn_mask(
                   lowerCamelCase							,      lowerCamelCase							,      lowerCamelCase							,      lowerCamelCase	)

 @slow
 def        __snake_case       (      self							:		List[str]	)			->				int:
        _UpperCamelCase  :							Any	    =	GPTaTokenizer.from_pretrained('gpt2'							,      pad_token='</s>'							,      padding_side='left'	)
        _UpperCamelCase  :							Tuple	    =	tokenizer(['Hello this is a long string', 'Hey']							,      return_tensors='np'							,      padding=lowerCamelCase							,      truncation=lowerCamelCase	)

        _UpperCamelCase  :							str	    =	FlaxGPTaLMHeadModel.from_pretrained('gpt2'	)
        _UpperCamelCase  :							Dict	    =	False
        _UpperCamelCase  :							Tuple	    =	model.config.eos_token_id

        _UpperCamelCase  :							Any	    =	jax.jit(model.generate	)

        _UpperCamelCase  :							Any	    =	jit_generate(inputs['input_ids']							,      attention_mask=inputs['attention_mask']	).sequences

        _UpperCamelCase  :							List[Any]	    =	tokenizer.batch_decode(lowerCamelCase							,      skip_special_tokens=lowerCamelCase	)

        _UpperCamelCase  :							Dict	    =	[
            'Hello this is a long string of words. I\'m going to start with the first one.\n',
            'Hey, I\'m not sure if I\'m going to be able to do',
        ]

        self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)

 @is_pt_flax_cross_test
 def        __snake_case       (      self							:		int	)			->				Optional[int]:
        _UpperCamelCase    ,   _UpperCamelCase  :							Optional[Any]	    =	self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               with self.subTest(model_class.__name__	):
                      # prepare inputs
                      _UpperCamelCase  :							Dict	    =	self._prepare_for_class(lowerCamelCase							,      lowerCamelCase	)
                      _UpperCamelCase  :							Union[str, Any]	    =	{k: torch.tensor(v.tolist()	) for k, v in prepared_inputs_dict.items()}

                      # load corresponding PyTorch class
                      _UpperCamelCase  :							Tuple	    =	model_class.__name__[4:]  # Skip the "Flax" at the beginning
                      _UpperCamelCase  :							Tuple	    =	getattr(lowerCamelCase							,      lowerCamelCase	)

                      _UpperCamelCase    ,   _UpperCamelCase  :							List[str]	    =	pt_inputs['input_ids'].shape
                      _UpperCamelCase  :							Optional[Any]	    =	np.random.randint(0							,      seq_length - 1							,      size=(batch_size,)	)
                      for batch_idx, start_index in enumerate(lowerCamelCase	):
                             _UpperCamelCase  :							Any	    =	0
                             _UpperCamelCase  :							Optional[Any]	    =	1
                             _UpperCamelCase  :							Optional[int]	    =	0
                             _UpperCamelCase  :							Optional[Any]	    =	1
                      _UpperCamelCase  :							int	    =	pt_model_class(lowerCamelCase	).eval()
                      _UpperCamelCase  :							Optional[int]	    =	model_class(lowerCamelCase							,      dtype=jnp.floataa	)

                      _UpperCamelCase  :							List[str]	    =	convert_pytorch_state_dict_to_flax(pt_model.state_dict()							,      lowerCamelCase	)
                      _UpperCamelCase  :							Tuple	    =	fx_state

                      with torch.no_grad():
                             _UpperCamelCase  :							Optional[int]	    =	pt_model(**lowerCamelCase	).to_tuple()

                      _UpperCamelCase  :							Any	    =	fx_model(**lowerCamelCase	).to_tuple()
                      self.assertEqual(len(lowerCamelCase	)							,      len(lowerCamelCase	)							,      'Output lengths differ between Flax and PyTorch'	)
                      for fx_output, pt_output in zip(lowerCamelCase							,      lowerCamelCase	):
                             self.assert_almost_equals(fx_output[:, -1]							,      pt_output[:, -1].numpy()							,      4e-2	)

                      with tempfile.TemporaryDirectory() as tmpdirname:
                             pt_model.save_pretrained(lowerCamelCase	)
                             _UpperCamelCase  :							Optional[Any]	    =	model_class.from_pretrained(lowerCamelCase							,      from_pt=lowerCamelCase	)

                      _UpperCamelCase  :							List[Any]	    =	fx_model_loaded(**lowerCamelCase	).to_tuple()
                      self.assertEqual(
                          len(lowerCamelCase	)							,      len(lowerCamelCase	)							,      'Output lengths differ between Flax and PyTorch'	)
                      for fx_output_loaded, pt_output in zip(lowerCamelCase							,      lowerCamelCase	):
                             self.assert_almost_equals(fx_output_loaded[:, -1]							,      pt_output[:, -1].numpy()							,      4e-2	)

 @is_pt_flax_cross_test
 def        __snake_case       (      self							:		Optional[Any]	)			->				int:
        _UpperCamelCase    ,   _UpperCamelCase  :							Any	    =	self.model_tester.prepare_config_and_inputs_for_common()
        for model_class in self.all_model_classes:
               with self.subTest(model_class.__name__	):
                      # prepare inputs
                      _UpperCamelCase  :							Optional[int]	    =	self._prepare_for_class(lowerCamelCase							,      lowerCamelCase	)
                      _UpperCamelCase  :							int	    =	{k: torch.tensor(v.tolist()	) for k, v in prepared_inputs_dict.items()}

                      # load corresponding PyTorch class
                      _UpperCamelCase  :							str	    =	model_class.__name__[4:]  # Skip the "Flax" at the beginning
                      _UpperCamelCase  :							Union[str, Any]	    =	getattr(lowerCamelCase							,      lowerCamelCase	)

                      _UpperCamelCase  :							Optional[Any]	    =	pt_model_class(lowerCamelCase	).eval()
                      _UpperCamelCase  :							str	    =	model_class(lowerCamelCase							,      dtype=jnp.floataa	)

                      _UpperCamelCase  :							Tuple	    =	load_flax_weights_in_pytorch_model(lowerCamelCase							,      fx_model.params	)
                      _UpperCamelCase    ,   _UpperCamelCase  :							List[Any]	    =	pt_inputs['input_ids'].shape
                      _UpperCamelCase  :							List[Any]	    =	np.random.randint(0							,      seq_length - 1							,      size=(batch_size,)	)
                      for batch_idx, start_index in enumerate(lowerCamelCase	):
                             _UpperCamelCase  :							Optional[int]	    =	0
                             _UpperCamelCase  :							Union[str, Any]	    =	1
                             _UpperCamelCase  :							Dict	    =	0
                             _UpperCamelCase  :							Union[str, Any]	    =	1

                      # make sure weights are tied in PyTorch
                      pt_model.tie_weights()

                      with torch.no_grad():
                             _UpperCamelCase  :							Optional[int]	    =	pt_model(**lowerCamelCase	).to_tuple()

                      _UpperCamelCase  :							Any	    =	fx_model(**lowerCamelCase	).to_tuple()
                      self.assertEqual(len(lowerCamelCase	)							,      len(lowerCamelCase	)							,      'Output lengths differ between Flax and PyTorch'	)
                      for fx_output, pt_output in zip(lowerCamelCase							,      lowerCamelCase	):
                             self.assert_almost_equals(fx_output[:, -1]							,      pt_output[:, -1].numpy()							,      4e-2	)

                      with tempfile.TemporaryDirectory() as tmpdirname:
                             fx_model.save_pretrained(lowerCamelCase	)
                             _UpperCamelCase  :							Dict	    =	pt_model_class.from_pretrained(lowerCamelCase							,      from_flax=lowerCamelCase	)

                      with torch.no_grad():
                             _UpperCamelCase  :							Tuple	    =	pt_model_loaded(**lowerCamelCase	).to_tuple()

                      self.assertEqual(
                          len(lowerCamelCase	)							,      len(lowerCamelCase	)							,      'Output lengths differ between Flax and PyTorch'	)
                      for fx_output, pt_output in zip(lowerCamelCase							,      lowerCamelCase	):
                             self.assert_almost_equals(fx_output[:, -1]							,      pt_output[:, -1].numpy()							,      4e-2	)

 @slow
 def        __snake_case       (      self							:		Union[str, Any]	)			->				Dict:
        for model_class_name in self.all_model_classes:
               _UpperCamelCase  :							List[Any]	    =	model_class_name.from_pretrained('gpt2'							,      from_pt=lowerCamelCase	)
               _UpperCamelCase  :							Tuple	    =	model(np.ones((1, 1)	)	)
               self.assertIsNotNone(lowerCamelCase	)





"""simple docstring"""

import datasets

from .evaluate import (
    apply_no_ans_threshold,
    find_all_best_thresh,
    get_raw_scores,
    make_eval_dict,
    make_qid_to_has_ans,
    merge_eval,
)


UpperCamelCase      :							List[str]          =					"\\n@inproceedings{Rajpurkar2016SQuAD10,\n  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},\n  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},\n  booktitle={EMNLP},\n  year={2016}\n}\n"

UpperCamelCase      :							int          =					"\nThis metric wrap the official scoring script for version 2 of the Stanford Question\nAnswering Dataset (SQuAD).\n\nStanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by\ncrowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,\nfrom the corresponding reading passage, or the question might be unanswerable.\n\nSQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions\nwritten adversarially by crowdworkers to look similar to answerable ones.\nTo do well on SQuAD2.0, systems must not only answer questions when possible, but also\ndetermine when no answer is supported by the paragraph and abstain from answering.\n"

UpperCamelCase      :							Optional[Any]          =					"\nComputes SQuAD v2 scores (F1 and EM).\nArgs:\n    predictions: List of triple for question-answers to score with the following elements:\n        - the question-answer 'id' field as given in the references (see below)\n        - the text of the answer\n        - the probability that the question has no answer\n    references: List of question-answers dictionaries with the following key-values:\n            - 'id': id of the question-answer pair (see above),\n            - 'answers': a list of Dict {'text': text of the answer as a string}\n    no_answer_threshold: float\n        Probability threshold to decide that a question has no answer.\nReturns:\n    'exact': Exact match (the normalized answer exactly match the gold answer)\n    'f1': The F-score of predicted tokens versus the gold answer\n    'total': Number of score considered\n    'HasAns_exact': Exact match (the normalized answer exactly match the gold answer)\n    'HasAns_f1': The F-score of predicted tokens versus the gold answer\n    'HasAns_total': Number of score considered\n    'NoAns_exact': Exact match (the normalized answer exactly match the gold answer)\n    'NoAns_f1': The F-score of predicted tokens versus the gold answer\n    'NoAns_total': Number of score considered\n    'best_exact': Best exact match (with varying threshold)\n    'best_exact_thresh': No-answer probability threshold associated to the best exact match\n    'best_f1': Best F1 (with varying threshold)\n    'best_f1_thresh': No-answer probability threshold associated to the best F1\nExamples:\n\n    >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}]\n    >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}]\n    >>> squad_v2_metric = datasets.load_metric(\"squad_v2\")\n    >>> results = squad_v2_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact': 100.0, 'f1': 100.0, 'total': 1, 'HasAns_exact': 100.0, 'HasAns_f1': 100.0, 'HasAns_total': 1, 'best_exact': 100.0, 'best_exact_thresh': 0.0, 'best_f1': 100.0, 'best_f1_thresh': 0.0}\n"

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,    _KWARGS_DESCRIPTION     )
class __lowerCAmelCase						( datasets.Metric     ):





     def 		UpperCAmelCase   (      self ):

      '''simple docstring'''




      return datasets.MetricInfo(
          description=_DESCRIPTION      ,	citation=_CITATION      ,	inputs_description=_KWARGS_DESCRIPTION      ,	features=datasets.Features(
              {
                  'predictions': {
                      'id': datasets.Value('string' ),
                      'prediction_text': datasets.Value('string' ),
                      'no_answer_probability': datasets.Value('float32' ),
                  },
                  'references': {
                      'id': datasets.Value('string' ),
                      'answers': datasets.features.Sequence(
                          {'text': datasets.Value('string' ), 'answer_start': datasets.Value('int32' )} ),
                  },
              } )      ,	codebase_urls=['https://rajpurkar.github.io/SQuAD-explorer/']      ,	reference_urls=['https://rajpurkar.github.io/SQuAD-explorer/']      ,	)







     def 		UpperCAmelCase   (      self      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase=1.0 ):

      '''simple docstring'''




      __UpperCamelCase							    =			{p['id']: p['no_answer_probability'] for p in predictions}
      __UpperCamelCase							    =			[{'paragraphs': [{'qas': references}]}]
      __UpperCamelCase							    =			{p['id']: p['prediction_text'] for p in predictions}

      __UpperCamelCase							    =			make_qid_to_has_ans(__UpperCAmelCase )  # maps qid to True/False
      __UpperCamelCase							    =			[k for k, v in qid_to_has_ans.items() if v]
      __UpperCamelCase							    =			[k for k, v in qid_to_has_ans.items() if not v]

      __UpperCamelCase    ,      __UpperCamelCase							    =			get_raw_scores(__UpperCAmelCase      ,	__UpperCAmelCase )
      __UpperCamelCase							    =			apply_no_ans_threshold(__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase )
      __UpperCamelCase							    =			apply_no_ans_threshold(__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase )
      __UpperCamelCase							    =			make_eval_dict(__UpperCAmelCase      ,	__UpperCAmelCase )

      if has_ans_qids:
       __UpperCamelCase							    =			make_eval_dict(__UpperCAmelCase      ,	__UpperCAmelCase      ,	qid_list=__UpperCAmelCase )
       merge_eval(__UpperCAmelCase      ,	__UpperCAmelCase      ,	'HasAns' )
      if no_ans_qids:
       __UpperCamelCase							    =			make_eval_dict(__UpperCAmelCase      ,	__UpperCAmelCase      ,	qid_list=__UpperCAmelCase )
       merge_eval(__UpperCAmelCase      ,	__UpperCAmelCase      ,	'NoAns' )
      find_all_best_thresh(__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase      ,	__UpperCAmelCase )
      return dict(__UpperCAmelCase )



