





import argparse
import json
import re
import string
import sys

import numpy as np


lowerCAmelCase_   =   0.5




def 						A_			(  lowercase_      ,		lowercase_  )			->						Optional[int]:
       _snake_case      :							int     				=						['''.''', ''',''', ''';''', ''':''']

       for token in remove_tokens:
              _snake_case      :							int     				=						ground_truth.replace(lowercase_      ,		''''''  )
              _snake_case      :							Tuple     				=						prediction.replace(lowercase_      ,		''''''  )

       _snake_case				,    _snake_case      :							List[str]     				=						ground_truth.lower(), prediction.lower()
       _snake_case				,    _snake_case      :							List[Any]     				=						ground_truth.replace('''/'''      ,		''' '''  ), prediction.replace('''/'''      ,		''' '''  )
       _snake_case				,    _snake_case      :							Tuple     				=						set(ground_truth.split(''' '''  )  ), set(prediction.split(''' '''  )  )

       _snake_case      :							Union[str, Any]     				=						ground_truth.intersection(lowercase_  )
       _snake_case      :							int     				=						ground_truth.union(lowercase_  )
       _snake_case      :							int     				=						len(lowercase_  ) / len(lowercase_  )
       return jaccard




def 						A_			(  lowercase_  )			->						Tuple:

       def remove_articles(lowercase_  ):
              return re.sub(r'''\b(a|an|the)\b'''      ,		''' '''      ,		lowercase_  )

       def white_space_fix(lowercase_  ):
              return " ".join(text.split()  )

       def remove_punc(lowercase_  ):
              _snake_case      :							List[str]     				=						set(string.punctuation  )
              return "".join(ch for ch in text if ch not in exclude  )

       def lower(lowercase_  ):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(lowercase_  )  )  )  )




def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						List[str]:
       _snake_case				,    _snake_case				,    _snake_case      :							Tuple     				=						0, 0, 0

       _snake_case      :							Optional[int]     				=						'''Parties''' in qa_id

       # first check if ground truth is empty
       if len(lowercase_  ) == 0:
              if len(lowercase_  ) > 0:
                     fp += len(lowercase_  )  # false positive for each one
       else:
              for ground_truth in ground_truths:
                     assert len(lowercase_  ) > 0
                     # check if there is a match
                     _snake_case      :							Optional[Any]     				=						False
                     for pred in predictions:
                            if substr_ok:
                                   _snake_case      :							List[Any]     				=						get_jaccard(lowercase_      ,		lowercase_  ) >= IOU_THRESH or ground_truth in pred
                            else:
                                   _snake_case      :							Any     				=						get_jaccard(lowercase_      ,		lowercase_  ) >= IOU_THRESH
                            if is_match:
                                   _snake_case      :							Union[str, Any]     				=						True

                     if match_found:
                            tp += 1
                     else:
                            fn += 1

        # now also get any fps by looping through preds
              for pred in predictions:
                     # Check if there's a match. if so, don't count (don't want to double count based on the above)
                     # but if there's no match, then this is a false positive.
                     # (Note: we get the true positives in the above loop instead of this loop so that we don't double count
                     # multiple predictions that are matched with the same answer.)
                     _snake_case      :							Tuple     				=						False
                     for ground_truth in ground_truths:
                            assert len(lowercase_  ) > 0
                            if substr_ok:
                                   _snake_case      :							str     				=						get_jaccard(lowercase_      ,		lowercase_  ) >= IOU_THRESH or ground_truth in pred
                            else:
                                   _snake_case      :							Dict     				=						get_jaccard(lowercase_      ,		lowercase_  ) >= IOU_THRESH
                            if is_match:
                                   _snake_case      :							Optional[Any]     				=						True

                     if not match_found:
                            fp += 1

       _snake_case      :							Dict     				=						tp / (tp + fp) if tp + fp > 0 else np.nan
       _snake_case      :							str     				=						tp / (tp + fn) if tp + fn > 0 else np.nan

       return precision, recall




def 						A_			(  lowercase_  )			->						str:
       _snake_case      :							Optional[Any]     				=						precisions[::-1]
       for i in range(1      ,		len(lowercase_  )  ):
              _snake_case      :							Any     				=						max(precision_best[i - 1]      ,		precision_best[i]  )
       _snake_case      :							Tuple     				=						precision_best[::-1]
       return precisions




def 						A_			(  lowercase_      ,		lowercase_  )			->						List[Any]:
       _snake_case      :							List[str]     				=						process_precisions(lowercase_  )
       _snake_case      :							Tuple     				=						np.trapz(lowercase_      ,		lowercase_  )
       if np.isnan(lowercase_  ):
              return 0
       return aupr




def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						List[str]:
       _snake_case      :							Optional[int]     				=						process_precisions(lowercase_  )
       _snake_case      :							Dict     				=						0
       for prec, recall in zip(lowercase_      ,		lowercase_  ):
              if recall >= recall_thresh:
                     _snake_case      :							Optional[Any]     				=						prec
                     break
       return prec_at_recall




def 						A_			(  lowercase_      ,		lowercase_  )			->						int:
       return normalize_answer(lowercase_  ) == normalize_answer(lowercase_  )




def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						Any:
       _snake_case      :							Any     				=						0
       for pred in predictions:
              for ground_truth in ground_truths:
                     _snake_case      :							Optional[int]     				=						metric_fn(lowercase_      ,		lowercase_  )
                     if score == 1:  # break the loop when one prediction matches the ground truth
                            break
              if score == 1:
                     break
       return score




def 						A_			(  lowercase_      ,		lowercase_  )			->						List[Any]:
       _snake_case      :							List[Any]     				=						0
       _snake_case      :							Any     				=						[]
       _snake_case      :							int     				=						[]
       for article in dataset:
              for paragraph in article["paragraphs"]:
                     for qa in paragraph["qas"]:
                            total += 1
                            if qa["id"] not in predictions:
                                   _snake_case      :							int     				=						'''Unanswered question ''' + qa['''id'''] + ''' will receive score 0.'''
                                   print(lowercase_      ,		file=sys.stderr  )
                                   continue
                            _snake_case      :							List[str]     				=						[x['''text'''] for x in qa['''answers''']]
                            _snake_case      :							str     				=						predictions[qa['''id''']]
                            _snake_case				,    _snake_case      :							List[str]     				=						compute_precision_recall(lowercase_      ,		lowercase_      ,		qa['''id''']  )

                            precisions.append(lowercase_  )
                            recalls.append(lowercase_  )

                            if precision == 0 and recall == 0:
                                   fa += 0
                            else:
                                   fa += 2 * (precision * recall) / (precision + recall)

                            exact_match += metric_max_over_ground_truths(lowercase_      ,		lowercase_      ,		lowercase_  )

       _snake_case      :							int     				=						[x for _, x in sorted(zip(lowercase_      ,		lowercase_  )  )]
       recalls.sort()

       _snake_case      :							Dict     				=						1_00.0 * fa / total
       _snake_case      :							str     				=						1_00.0 * exact_match / total
       _snake_case      :							Tuple     				=						get_aupr(lowercase_      ,		lowercase_  )

       _snake_case      :							Optional[Any]     				=						get_prec_at_recall(lowercase_      ,		lowercase_      ,		recall_thresh=0.9  )
       _snake_case      :							Dict     				=						get_prec_at_recall(lowercase_      ,		lowercase_      ,		recall_thresh=0.8  )

       return {
           "exact_match": exact_match,
           "f1": fa,
           "aupr": aupr,
           "prec_at_80_recall": prec_at_aa_recall,
           "prec_at_90_recall": prec_at_aa_recall,
       }


if __name__ == "__main__":
   lowerCAmelCase_   =   argparse.ArgumentParser(description="Evaluation for CUAD")
   parser.add_argument("dataset_file", help="Dataset file")
   parser.add_argument("prediction_file", help="Prediction File")
   lowerCAmelCase_   =   parser.parse_args()
   with open(args.dataset_file) as dataset_file:
      lowerCAmelCase_   =   json.load(dataset_file)
      lowerCAmelCase_   =   dataset_json["data"]
   with open(args.prediction_file) as prediction_file:
      lowerCAmelCase_   =   json.load(prediction_file)
   print(json.dumps(evaluate(dataset, predictions)))













from functools import partial
from typing import Any, Callable, Dict, List, Type, TypeVar, Union, overload

import torch
import torch.nn as nn
import torch.types




def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						torch.Tensor:
       # The first operation in a checkpoint can't be in-place, but it's
       # nice to have in-place addition during inference. Thus...
       if not inplace:
              _snake_case      :							Any     				=						ma + ma
       else:
              ma += ma

       return ma




def 						A_			(  lowercase_      ,		lowercase_  )			->						torch.Tensor:
       _snake_case      :							str     				=						-1 * len(lowercase_  )
       _snake_case      :							Any     				=						list(range(len(tensor.shape[:zero_index]  )  )  )
       return tensor.permute(first_inds + [zero_index + i for i in inds]  )




def 						A_			(  lowercase_      ,		lowercase_  )			->						torch.Tensor:
       return t.reshape(t.shape[:-no_dims] + (-1,)  )




def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_      ,		lowercase_ = 1E-4  )			->						torch.Tensor:
       _snake_case      :							Any     				=						mask.expand(*value.shape  )
       return torch.sum(mask * value      ,		dim=lowercase_  ) / (eps + torch.sum(lowercase_      ,		dim=lowercase_  ))




def 						A_			(  lowercase_      ,		lowercase_ = 2.31_25      ,		lowercase_ = 21.68_75      ,		lowercase_ = 64  )			->						torch.Tensor:
       _snake_case      :							int     				=						torch.linspace(lowercase_      ,		lowercase_      ,		no_bins - 1      ,		device=pts.device  )
       _snake_case      :							Optional[Any]     				=						torch.sqrt(torch.sum((pts.unsqueeze(-2  ) - pts.unsqueeze(-3  )) ** 2      ,		dim=-1  )  )
       return torch.bucketize(lowercase_      ,		lowercase_  )




def 						A_			(  lowercase_      ,		lowercase_  )			->						dict:
       _snake_case      :							Dict     				=						dicts[0]
       _snake_case      :							str     				=						{}
       for k, v in first.items():
              _snake_case      :							List[Any]     				=						[d[k] for d in dicts]
              if isinstance(lowercase_      ,		lowercase_  ):
                     _snake_case      :							Any     				=						dict_multimap(lowercase_      ,		lowercase_  )
              else:
                     _snake_case      :							Dict     				=						fn(lowercase_  )

       return new_dict




def 						A_			(  lowercase_      ,		lowercase_  )			->						torch.Tensor:
       _snake_case      :							str     				=						v_bins.view(((1,) * len(x.shape  )) + (len(lowercase_  ),)  )
       _snake_case      :							Union[str, Any]     				=						x[..., None] - reshaped_bins
       _snake_case      :							Optional[Any]     				=						torch.argmin(torch.abs(lowercase_  )      ,		dim=-1  )
       return nn.functional.one_hot(lowercase_      ,		num_classes=len(lowercase_  )  ).float()




def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_ = 0      ,		lowercase_ = 0  )			->						torch.Tensor:
       _snake_case      :							List[Union[slice, torch.Tensor]]     				=						[]
       for i, s in enumerate(data.shape[:no_batch_dims]  ):
              _snake_case      :							Optional[Any]     				=						torch.arange(lowercase_  )
              _snake_case      :							Union[str, Any]     				=						r.view(*(*((1,) * i), -1, *((1,) * (len(inds.shape  ) - i - 1)))  )
              ranges.append(lowercase_  )

       _snake_case      :							List[Union[slice, torch.Tensor]]     				=						[slice(lowercase_  ) for _ in range(len(data.shape  ) - no_batch_dims  )]
       _snake_case      :							Union[str, Any]     				=						inds
       ranges.extend(lowercase_  )
       # Matt note: Editing this to get around the behaviour of using a list as an array index changing
       # in recent Numpy versions
       return data[tuple(lowercase_  )]


lowerCAmelCase_   =   TypeVar("T")




def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						Dict[Any, Union[dict, list, tuple, Any]]:
       _snake_case      :							Dict[Any, Union[dict, list, tuple, Any]]     				=						{}
       for k, v in dic.items():
              if isinstance(lowercase_      ,		lowercase_  ):
                     _snake_case      :							Dict     				=						dict_map(lowercase_      ,		lowercase_      ,		lowercase_  )
              else:
                     _snake_case      :							List[Any]     				=						tree_map(lowercase_      ,		lowercase_      ,		lowercase_  )

       return new_dict




@overload
def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						Any:
       ...




@overload
def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						dict:
       ...




@overload
def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						list:
       ...




@overload
def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						tuple:
       ...




def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						Any:
       if isinstance(lowercase_      ,		lowercase_  ):
              return dict_map(lowercase_      ,		lowercase_      ,		lowercase_  )
       elif isinstance(lowercase_      ,		lowercase_  ):
              return [tree_map(lowercase_      ,		lowercase_      ,		lowercase_  ) for x in tree]
       elif isinstance(lowercase_      ,		lowercase_  ):
              return tuple(tree_map(lowercase_      ,		lowercase_      ,		lowercase_  ) for x in tree  )
       elif isinstance(lowercase_      ,		lowercase_  ):
              return fn(lowercase_  )
       else:
              print(type(lowercase_  )  )
              raise ValueError('''Not supported'''  )
lowerCAmelCase_   =   partial(tree_map, leaf_type=torch.Tensor)






