





import logging
import os
from dataclasses import dataclass, field
from typing import Dict, List, Optional

import numpy as np
import torch
from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from transformers.trainer_utils import is_main_process


UpperCamelCase__ :			str       	=			logging.getLogger(__name__)







@dataclass
class 	_UpperCamelCase	:







					'''simple docstring'''

					lowerCamelCase  :  str           =							field(
					    metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}	)
					lowerCamelCase  :  Optional[str]           =							field(
					    default=A_    ,							metadata={'help': 'Pretrained config name or path if not the same as model_name'}	)
					lowerCamelCase  :  Optional[str]           =							field(
					    default=A_    ,							metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}	)
					lowerCamelCase  :  Optional[str]           =							field(
					    default=A_    ,							metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}    ,							)







@dataclass
class 	_UpperCamelCase	:







					'''simple docstring'''

					lowerCamelCase  :  str           =							field(
					    metadata={'help': 'The name of the task to train selected in the list: ' + ', '.join(hans_processors.keys()	)}	)
					lowerCamelCase  :  str           =							field(
					    metadata={'help': 'The input data dir. Should contain the .tsv files (or other data files) for the task.'}	)
					lowerCamelCase  :  int           =							field(
					    default=128    ,							metadata={
					        'help': (
					            'The maximum total input sequence length after tokenization. Sequences longer '
					            'than this will be truncated, sequences shorter will be padded.'
					        )
					    }    ,							)
					lowerCamelCase  :  bool           =							field(
					    default=A_    ,							metadata={'help': 'Overwrite the cached training and evaluation sets'}	)




def  A_(      A ):
	UpperCAmelCase_		    =  default_data_collator(A )
	UpperCAmelCase_		    =  batch.pop("""pairID"""					,   A )
	return batch




def  A_(      ):
	# See all possible arguments in src/transformers/training_args.py
	# or by passing the --help flag to this script.
	# We now keep distinct sets of args, for a cleaner separation of concerns.

	UpperCAmelCase_		    =  HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )
	UpperCAmelCase_    ,						UpperCAmelCase_    ,						UpperCAmelCase_		    =  parser.parse_args_into_dataclasses()

	if (
	    os.path.exists(training_args.output_dir )
	    and os.listdir(training_args.output_dir )
	    and training_args.do_train
	    and not training_args.overwrite_output_dir
	):
		raise ValueError(
		    f"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
		    """ --overwrite_output_dir to overcome.""" )

	# Setup logging
	logging.basicConfig(
	    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""					,   datefmt="""%m/%d/%Y %H:%M:%S"""					,   level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN					,   )
	logger.warning(
	    """Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"""					,   training_args.local_rank					,   training_args.device					,   training_args.n_gpu					,   bool(training_args.local_rank != -1 )					,   training_args.fpaa					,   )
	# Set the verbosity to info of the Transformers logger (on main process only):
	if is_main_process(training_args.local_rank ):
		transformers.utils.logging.set_verbosity_info()
		transformers.utils.logging.enable_default_handler()
		transformers.utils.logging.enable_explicit_format()
	logger.info("""Training/evaluation parameters %s"""					,   A )

	# Set seed
	set_seed(training_args.seed )

	try:
		UpperCAmelCase_		    =  hans_tasks_num_labels[data_args.task_name]
	except KeyError:
		raise ValueError("""Task not found: %s""" % (data_args.task_name) )

	# Load pretrained model and tokenizer
	#
	# Distributed training:
	# The .from_pretrained methods guarantee that only one local process can concurrently
	# download model & vocab.

	UpperCAmelCase_		    =  AutoConfig.from_pretrained(
	    model_args.config_name if model_args.config_name else model_args.model_name_or_path					,   num_labels=A					,   finetuning_task=data_args.task_name					,   cache_dir=model_args.cache_dir					,   )
	UpperCAmelCase_		    =  AutoTokenizer.from_pretrained(
	    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path					,   cache_dir=model_args.cache_dir					,   )
	UpperCAmelCase_		    =  AutoModelForSequenceClassification.from_pretrained(
	    model_args.model_name_or_path					,   from_tf=bool(""".ckpt""" in model_args.model_name_or_path )					,   config=A					,   cache_dir=model_args.cache_dir					,   )

	# Get datasets
	UpperCAmelCase_		    =  (
	    HansDataset(
	        data_dir=data_args.data_dir					,   tokenizer=A					,   task=data_args.task_name					,   max_seq_length=data_args.max_seq_length					,   overwrite_cache=data_args.overwrite_cache					,   )
	    if training_args.do_train
	    else None
	)
	UpperCAmelCase_		    =  (
	    HansDataset(
	        data_dir=data_args.data_dir					,   tokenizer=A					,   task=data_args.task_name					,   max_seq_length=data_args.max_seq_length					,   overwrite_cache=data_args.overwrite_cache					,   evaluate=A					,   )
	    if training_args.do_eval
	    else None
	)

	# Initialize our Trainer
	UpperCAmelCase_		    =  Trainer(
	    model=A					,   args=A					,   train_dataset=A					,   eval_dataset=A					,   data_collator=A					,   )

	# Training
	if training_args.do_train:
		trainer.train(
		    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path ) else None )
		trainer.save_model()
		# For convenience, we also re-save the tokenizer to the same directory,
		# so that you can share your model easily on huggingface.co/models =)
		if trainer.is_world_master():
			tokenizer.save_pretrained(training_args.output_dir )

    # Evaluation
	if training_args.do_eval:
		logger.info("""*** Evaluate ***""" )

		UpperCAmelCase_		    =  trainer.predict(A )
		UpperCAmelCase_		    =  output.predictions
		UpperCAmelCase_		    =  np.argmax(A					,   axis=1 )

		UpperCAmelCase_		    =  [ex.pairID for ex in eval_dataset]
		UpperCAmelCase_		    =  os.path.join(training_args.output_dir					,   """hans_predictions.txt""" )
		UpperCAmelCase_		    =  eval_dataset.get_labels()
		if trainer.is_world_master():
			with open(A					,   """w""" ) as writer:
				writer.write("""pairID,gold_label\n""" )
				for pid, pred in zip(A					,   A ):
					writer.write("""ex""" + str(A ) + """,""" + label_list[int(A )] + """\n""" )

		trainer._log(output.metrics )




def  A_(      A ):
	# For xla_spawn (TPUs)
	main()


if __name__ == "__main__":
				main()









import argparse
import json
import re
import string
import sys
from collections import Counter




def  A_(      A ):

	def remove_articles(A ):
		return re.sub(R"""\b(a|an|the)\b"""					,   """ """					,   A )

	def white_space_fix(A ):
		return " ".join(text.split() )

	def remove_punc(A ):
		UpperCAmelCase_		    =  set(string.punctuation )
		return "".join(ch for ch in text if ch not in exclude )

	def lower(A ):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(A ) ) ) )




def  A_(      A					,   A ):
	UpperCAmelCase_		    =  normalize_answer(A ).split()
	UpperCAmelCase_		    =  normalize_answer(A ).split()
	UpperCAmelCase_		    =  Counter(A ) & Counter(A )
	UpperCAmelCase_		    =  sum(common.values() )
	if num_same == 0:
		return 0
	UpperCAmelCase_		    =  1.0 * num_same / len(A )
	UpperCAmelCase_		    =  1.0 * num_same / len(A )
	UpperCAmelCase_		    =  (2 * precision * recall) / (precision + recall)
	return fa




def  A_(      A					,   A ):
	return normalize_answer(A ) == normalize_answer(A )




def  A_(      A					,   A					,   A ):
	UpperCAmelCase_		    =  []
	for ground_truth in ground_truths:
		UpperCAmelCase_		    =  metric_fn(A					,   A )
		scores_for_ground_truths.append(A )
	return max(A )




def  A_(      A					,   A ):
	UpperCAmelCase_		    =  UpperCAmelCase_		    =  UpperCAmelCase_		    =  0
	UpperCAmelCase_		    =  []
	for passage in dataset:
		for qa in passage["qas"]:
			total += 1
			if qa["id"] not in predictions:
				UpperCAmelCase_		    =  f"""Unanswered question {qa['id']} will receive score 0."""
				print(A					,   file=sys.stderr )
				continue

			UpperCAmelCase_		    =  [x["""text"""] for x in qa["""answers"""]]
			UpperCAmelCase_		    =  predictions[qa["""id"""]]

			UpperCAmelCase_		    =  metric_max_over_ground_truths(A					,   A					,   A )
			if int(_exact_match ) == 1:
				correct_ids.append(qa["""id"""] )
			exact_match += _exact_match

			fa += metric_max_over_ground_truths(A					,   A					,   A )

	UpperCAmelCase_		    =  exact_match / total
	UpperCAmelCase_		    =  fa / total

	return {"exact_match": exact_match, "f1": fa}, correct_ids


if __name__ == "__main__":
				UpperCamelCase__ :			Any       	=			"""1.0"""
				UpperCamelCase__ :			Optional[Any]       	=			argparse.ArgumentParser("""Official evaluation script for ReCoRD v1.0.""")
				parser.add_argument("""data_file""", help="""The dataset file in JSON format.""")
				parser.add_argument("""pred_file""", help="""The model prediction file in JSON format.""")
				parser.add_argument("""--output_correct_ids""", action="""store_true""", help="""Output the correctly answered query IDs.""")
				UpperCamelCase__ :			Any       	=			parser.parse_args()

				with open(args.data_file) as data_file:
								UpperCamelCase__ :			int       	=			json.load(data_file)
								if dataset_json["version"] != expected_version:
												print(
												    f"Evaluation expects v-{expected_version}, but got dataset with v-{dataset_json['version']}",
												    file=sys.stderr,
												)
								UpperCamelCase__ :			str       	=			dataset_json["""data"""]

				with open(args.pred_file) as pred_file:
								UpperCamelCase__ :			List[Any]       	=			json.load(pred_file)

				UpperCamelCase__	,      UpperCamelCase__ :			Optional[int]       	=			evaluate(dataset, predictions)

				if args.output_correct_ids:
								print(f"Output {len(correct_ids)} correctly answered question IDs.")
								with open("""correct_ids.json""", """w""") as f:
												json.dump(correct_ids, f)


