




'''simple docstring'''




import csv
import json
import logging
import os
import re
import subprocess
import tempfile
import unittest
import zipfile
from pathlib import Path
from typing import Optional
from unittest import mock

import torch

# We use TF to parse the logs
from accelerate import Accelerator
from accelerate.test_utils.testing import (
    MockingTestCase,
    TempDirTestCase,
    require_comet_ml,
    require_tensorboard,
    require_wandb,
    skip,
)
from accelerate.tracking import CometMLTracker, GeneralTracker
from accelerate.utils import ProjectConfiguration, is_comet_ml_available, is_tensorboard_available


if is_comet_ml_available():
							from comet_ml import OfflineExperiment

if is_tensorboard_available():
							import struct

							import tensorboard.compat.proto.event_pba as event_pba

snake_case__ 							=    logging.getLogger(__name__)






@require_tensorboard
class      UpperCamelCase_  (unittest.TestCase  ):







						"""simple docstring"""


						def    _a     (  self   :							Dict     ):





											"""simple docstring"""







											A_			:  List[str]			=				'''test_project_with_config'''
											with tempfile.TemporaryDirectory() as dirpath:
																A_			:  List[str]			=				Accelerator(log_with='''tensorboard'''				,			project_dir=_lowerCamelCase     )
																A_			:  List[str]			=				{'''num_iterations''': 12, '''learning_rate''': 1E-2, '''some_boolean''': False, '''some_string''': '''some_value'''}
																accelerator.init_trackers(_lowerCamelCase				,			_lowerCamelCase     )
																accelerator.end_training()
																for child in Path(f'{dirpath}/{project_name}'     ).glob('''*/**'''     ):
																					A_			:  List[Any]			=				list(filter(lambda _lowerCamelCase     : x.is_file()				,			child.iterdir()     )     )[0]
																self.assertNotEqual(str(_lowerCamelCase     )				,			''''''     )







						def    _a     (  self   :							Union[str, Any]     ):





											"""simple docstring"""







											A_			:  Tuple			=				'''test_project_with_log'''
											with tempfile.TemporaryDirectory() as dirpath:
																A_			:  str			=				Accelerator(log_with='''tensorboard'''				,			project_dir=_lowerCamelCase     )
																accelerator.init_trackers(_lowerCamelCase     )
																A_			:  str			=				{'''total_loss''': 0.1, '''iteration''': 1, '''my_text''': '''some_value'''}
																accelerator.log(_lowerCamelCase				,			step=0     )
																accelerator.end_training()
																# Logged values are stored in the outermost-tfevents file and can be read in as a TFRecord
																# Names are randomly generated each time
																A_			:  int			=				list(filter(lambda _lowerCamelCase     : x.is_file()				,			Path(f'{dirpath}/{project_name}'     ).iterdir()     )     )[0]
																self.assertNotEqual(str(_lowerCamelCase     )				,			''''''     )







						def    _a     (  self   :							Dict     ):





											"""simple docstring"""







											A_			:  Tuple			=				'''test_project_with_log'''
											with tempfile.TemporaryDirectory() as dirpath:
																A_			:  Tuple			=				Accelerator(log_with='''tensorboard'''				,			project_dir=_lowerCamelCase     )
																accelerator.init_trackers(_lowerCamelCase     )
																A_			:  str			=				{'''tensor''': torch.tensor(1     )}
																accelerator.log(_lowerCamelCase				,			step=0     )
																accelerator.end_training()
																# Logged values are stored in the outermost-tfevents file and can be read in as a TFRecord
																# Names are randomly generated each time
																A_			:  Optional[Any]			=				list(filter(lambda _lowerCamelCase     : x.is_file()				,			Path(f'{dirpath}/{project_name}'     ).iterdir()     )     )[0]
																# Reading implementation based on https://github.com/pytorch/pytorch/issues/45327#issuecomment-703757685
																with open(_lowerCamelCase				,			'''rb'''     ) as f:
																					A_			:  int			=				f.read()
																A_			:  Union[str, Any]			=				False
																while data:
																					A_			:  int			=				struct.unpack('''Q'''				,			data[:8]     )

																					A_			:  Tuple			=				data[12 : 12 + int(header[0]     )]  # 8+4
																					A_			:  int			=				data[12 + int(header[0]     ) + 4 :]
																					A_			:  List[str]			=				event_pba.Event()

																					event.ParseFromString(_lowerCamelCase     )
																					if event.HasField('''summary'''     ):
																										for value in event.summary.value:
																															if value.simple_value == 1.0 and value.tag == "tensor":
																																				A_			:  str			=				True
																self.assertTrue(_lowerCamelCase				,			'''Converted tensor was not found in the log file!'''     )







						def    _a     (  self   :							Optional[Any]     ):





											"""simple docstring"""







											with self.assertRaisesRegex(_lowerCamelCase				,			'''Logging with `tensorboard` requires a `logging_dir`'''     ):
																A_			:  Union[str, Any]			=				Accelerator(log_with='''tensorboard'''     )
											with tempfile.TemporaryDirectory() as dirpath:
																A_			:  List[str]			=				Accelerator(log_with='''tensorboard'''				,			project_dir=_lowerCamelCase     )







						def    _a     (  self   :							str     ):





											"""simple docstring"""







											A_			:  List[str]			=				ProjectConfiguration(total_limit=30     )
											with tempfile.TemporaryDirectory() as dirpath:
																A_			:  Union[str, Any]			=				Accelerator(log_with='''tensorboard'''				,			project_dir=_lowerCamelCase				,			project_config=_lowerCamelCase     )






@require_wandb
@mock.patch.dict(os.environ,   {'WANDB_MODE': 'offline'}  )
class      UpperCamelCase_  (a__,   a__  ):







						"""simple docstring"""


						def    _a     (  self   :							Tuple     ):





											"""simple docstring"""







											super().setUp()
											# wandb let's us override where logs are stored to via the WANDB_DIR env var
											self.add_mocks(mock.patch.dict(os.environ				,			{'''WANDB_DIR''': self.tmpdir}     )     )







						@staticmethod
						def    _a     (  _lowerCamelCase   :							str				,			_lowerCamelCase   :							str				,			_lowerCamelCase   :							bool = True     ):





											"""simple docstring"""







											A_			:  List[Any]			=				Rf'{section} ([\S\s]*?)\n\n'
											if record:
																A_			:  List[Any]			=				Rf'Record: {pattern}'
											A_			:  Any			=				re.findall(_lowerCamelCase				,			_lowerCamelCase     )[0]
											# A config
											if section == "config" or section == "history":
																A_			:  int			=				re.findall(R'''"([a-zA-Z0-9_.,]+)'''				,			_lowerCamelCase     )
																return {key: val for key, val in zip(cleaned_record[0::2]				,			cleaned_record[1::2]     )}
											# Everything else
											else:
																return dict(re.findall(R'''(\w+): "([^\s]+)"'''				,			_lowerCamelCase     )     )







						@skip
						def    _a     (  self   :							Optional[int]     ):





											"""simple docstring"""







											A_			:  Union[str, Any]			=				'''test_project_with_config'''
											A_			:  List[str]			=				Accelerator(log_with='''wandb'''     )
											A_			:  List[Any]			=				{'''num_iterations''': 12, '''learning_rate''': 1E-2, '''some_boolean''': False, '''some_string''': '''some_value'''}
											A_			:  Tuple			=				{'''wandb''': {'''tags''': ['''my_tag''']}}
											accelerator.init_trackers(_lowerCamelCase				,			_lowerCamelCase				,			_lowerCamelCase     )
											A_			:  List[str]			=				{'''total_loss''': 0.1, '''iteration''': 1, '''my_text''': '''some_value'''}
											accelerator.log(_lowerCamelCase				,			step=0     )
											accelerator.end_training()
											# The latest offline log is stored at wandb/latest-run/*.wandb
											for child in Path(f'{self.tmpdir}/wandb/latest-run'     ).glob('''*'''     ):
																if child.is_file() and child.suffix == ".wandb":
																					A_			:  Dict			=				subprocess.check_output(
																					    ['''wandb''', '''sync''', '''--view''', '''--verbose''', str(_lowerCamelCase     )]				,			env=os.environ.copy()     ).decode('''utf8'''				,			'''ignore'''     )
																					break

        # Check HPS through careful parsing and cleaning
											A_			:  Tuple			=				self.parse_log(_lowerCamelCase				,			'''config'''     )
											self.assertEqual(logged_items['''num_iterations''']				,			'''12'''     )
											self.assertEqual(logged_items['''learning_rate''']				,			'''0.01'''     )
											self.assertEqual(logged_items['''some_boolean''']				,			'''false'''     )
											self.assertEqual(logged_items['''some_string''']				,			'''some_value'''     )
											self.assertEqual(logged_items['''some_string''']				,			'''some_value'''     )

											# Run tags
											A_			:  Tuple			=				self.parse_log(_lowerCamelCase				,			'''run'''				,			_lowerCamelCase     )
											self.assertEqual(logged_items['''tags''']				,			'''my_tag'''     )

											# Actual logging
											A_			:  Union[str, Any]			=				self.parse_log(_lowerCamelCase				,			'''history'''     )
											self.assertEqual(logged_items['''total_loss''']				,			'''0.1'''     )
											self.assertEqual(logged_items['''iteration''']				,			'''1'''     )
											self.assertEqual(logged_items['''my_text''']				,			'''some_value'''     )
											self.assertEqual(logged_items['''_step''']				,			'''0'''     )






def     snake_case__     (			self					: Optional[Any]      ,		lowerCamelCase__					: str      ,		lowerCamelCase__					: str      ) -> str:
					A_			:  Optional[Any]			=				run_name
					A_			:  Union[str, Any]			=				OfflineExperiment(project_name=lowerCamelCase__      ,		offline_directory=lowerCamelCase__      )
					logger.info(f'Initialized offline CometML project {self.run_name}'      )
					logger.info('''Make sure to log any initial configurations with `self.store_init_configuration` before training!'''      )






@require_comet_ml
@mock.patch.object(a__,   '__init__',   a__  )
class      UpperCamelCase_  (unittest.TestCase  ):







						"""simple docstring"""


						@staticmethod
						def    _a     (  _lowerCamelCase   :							str				,			_lowerCamelCase   :							str				,			_lowerCamelCase   :							bool = False     ):





											"""simple docstring"""







											for log in log_list:
																A_			:  Dict			=				json.loads(_lowerCamelCase     )['''payload''']
																if is_param and "param" in j.keys():
																					if j["param"]["paramName"] == key:
																										return j["param"]["paramValue"]
																if "log_other" in j.keys():
																					if j["log_other"]["key"] == key:
																										return j["log_other"]["val"]
																if "metric" in j.keys():
																					if j["metric"]["metricName"] == key:
																										return j["metric"]["metricValue"]







						def    _a     (  self   :							Any     ):





											"""simple docstring"""







											with tempfile.TemporaryDirectory() as d:
																A_			:  Optional[int]			=				CometMLTracker('''test_project_with_config'''				,			_lowerCamelCase     )
																A_			:  Any			=				Accelerator(log_with=_lowerCamelCase     )
																A_			:  Optional[int]			=				{'''num_iterations''': 12, '''learning_rate''': 1E-2, '''some_boolean''': False, '''some_string''': '''some_value'''}
																accelerator.init_trackers(_lowerCamelCase				,			_lowerCamelCase     )
																accelerator.end_training()
																A_			:  int			=				os.listdir(_lowerCamelCase     )[0]  # Comet is nice, it's just a zip file here
																# We parse the raw logs
																A_			:  Optional[Any]			=				os.path.join(_lowerCamelCase				,			_lowerCamelCase     )
																A_			:  int			=				zipfile.ZipFile(_lowerCamelCase				,			'''r'''     )
																A_			:  Any			=				archive.open('''messages.json'''     ).read().decode('''utf-8'''     )
											A_			:  List[Any]			=				log.split('''\n'''     )[:-1]
											self.assertEqual(self.get_value_from_key(_lowerCamelCase				,			'''num_iterations'''				,			_lowerCamelCase     )				,			12     )
											self.assertEqual(self.get_value_from_key(_lowerCamelCase				,			'''learning_rate'''				,			_lowerCamelCase     )				,			0.01     )
											self.assertEqual(self.get_value_from_key(_lowerCamelCase				,			'''some_boolean'''				,			_lowerCamelCase     )				,			_lowerCamelCase     )
											self.assertEqual(self.get_value_from_key(_lowerCamelCase				,			'''some_string'''				,			_lowerCamelCase     )				,			'''some_value'''     )







						def    _a     (  self   :							Tuple     ):





											"""simple docstring"""







											with tempfile.TemporaryDirectory() as d:
																A_			:  List[str]			=				CometMLTracker('''test_project_with_config'''				,			_lowerCamelCase     )
																A_			:  str			=				Accelerator(log_with=_lowerCamelCase     )
																accelerator.init_trackers(_lowerCamelCase     )
																A_			:  Optional[int]			=				{'''total_loss''': 0.1, '''iteration''': 1, '''my_text''': '''some_value'''}
																accelerator.log(_lowerCamelCase				,			step=0     )
																accelerator.end_training()
																A_			:  Optional[Any]			=				os.listdir(_lowerCamelCase     )[0]  # Comet is nice, it's just a zip file here
																# We parse the raw logs
																A_			:  Optional[int]			=				os.path.join(_lowerCamelCase				,			_lowerCamelCase     )
																A_			:  Any			=				zipfile.ZipFile(_lowerCamelCase				,			'''r'''     )
																A_			:  Optional[Any]			=				archive.open('''messages.json'''     ).read().decode('''utf-8'''     )
											A_			:  List[str]			=				log.split('''\n'''     )[:-1]
											self.assertEqual(self.get_value_from_key(_lowerCamelCase				,			'''curr_step'''				,			_lowerCamelCase     )				,			0     )
											self.assertEqual(self.get_value_from_key(_lowerCamelCase				,			'''total_loss'''     )				,			0.1     )
											self.assertEqual(self.get_value_from_key(_lowerCamelCase				,			'''iteration'''     )				,			1     )
											self.assertEqual(self.get_value_from_key(_lowerCamelCase				,			'''my_text'''     )				,			'''some_value'''     )






class      UpperCamelCase_  (a__  ):







						"""simple docstring"""


						_lowerCAmelCase         =       [
						    'total_loss',
						    'iteration',
						    'my_text',
						    'learning_rate',
						    'num_iterations',
						    'some_boolean',
						    'some_string',
						]

						_lowerCAmelCase         =       'my_custom_tracker'
						_lowerCAmelCase         =       False







						def __init__(  self   :							Optional[Any]				,			_lowerCamelCase   :							str     ):





											"""simple docstring"""







											A_			:  Tuple			=				open(f'{dir}/log.csv'				,			'''w+'''     )
											A_			:  Optional[int]			=				csv.DictWriter(self.f				,			fieldnames=self._col_names     )
											self.writer.writeheader()







						@property
						def    _a     (  self   :							List[str]     ):





											"""simple docstring"""







											return self.writer







						def    _a     (  self   :							Union[str, Any]				,			_lowerCamelCase   :							dict     ):





											"""simple docstring"""







											logger.info('''Call init'''     )
											self.writer.writerow(_lowerCamelCase     )







						def    _a     (  self   :							Tuple				,			_lowerCamelCase   :							dict				,			_lowerCamelCase   :							Optional[int]     ):





											"""simple docstring"""







											logger.info('''Call log'''     )
											self.writer.writerow(_lowerCamelCase     )







						def    _a     (  self   :							str     ):





											"""simple docstring"""







											self.f.close()






class      UpperCamelCase_  (unittest.TestCase  ):







						"""simple docstring"""


						def    _a     (  self   :							Optional[Any]     ):





											"""simple docstring"""







											with tempfile.TemporaryDirectory() as d:
																A_			:  Tuple			=				MyCustomTracker(_lowerCamelCase     )
																A_			:  Tuple			=				Accelerator(log_with=_lowerCamelCase     )
																A_			:  List[Any]			=				{'''num_iterations''': 12, '''learning_rate''': 1E-2, '''some_boolean''': False, '''some_string''': '''some_value'''}
																accelerator.init_trackers('''Some name'''				,			_lowerCamelCase     )
																accelerator.end_training()
																with open(f'{d}/log.csv'				,			'''r'''     ) as f:
																					A_			:  List[str]			=				csv.DictReader(_lowerCamelCase     )
																					A_			:  Optional[Any]			=				next(_lowerCamelCase     )
																					A_			:  Optional[Any]			=				{
																					    '''total_loss''': '''''',
																					    '''iteration''': '''''',
																					    '''my_text''': '''''',
																					    '''learning_rate''': '''0.01''',
																					    '''num_iterations''': '''12''',
																					    '''some_boolean''': '''False''',
																					    '''some_string''': '''some_value''',
																					}
																					self.assertDictEqual(_lowerCamelCase				,			_lowerCamelCase     )







						def    _a     (  self   :							List[str]     ):





											"""simple docstring"""







											with tempfile.TemporaryDirectory() as d:
																A_			:  Union[str, Any]			=				MyCustomTracker(_lowerCamelCase     )
																A_			:  Optional[Any]			=				Accelerator(log_with=_lowerCamelCase     )
																accelerator.init_trackers('''Some name'''     )
																A_			:  Optional[int]			=				{'''total_loss''': 0.1, '''iteration''': 1, '''my_text''': '''some_value'''}
																accelerator.log(_lowerCamelCase				,			step=0     )
																accelerator.end_training()
																with open(f'{d}/log.csv'				,			'''r'''     ) as f:
																					A_			:  Union[str, Any]			=				csv.DictReader(_lowerCamelCase     )
																					A_			:  Optional[Any]			=				next(_lowerCamelCase     )
																					A_			:  Dict			=				{
																					    '''total_loss''': '''0.1''',
																					    '''iteration''': '''1''',
																					    '''my_text''': '''some_value''',
																					    '''learning_rate''': '''''',
																					    '''num_iterations''': '''''',
																					    '''some_boolean''': '''''',
																					    '''some_string''': '''''',
																					}
																					self.assertDictEqual(_lowerCamelCase				,			_lowerCamelCase     )









'''simple docstring'''




import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


snake_case__ 							=    {0: """null""", 1: """short""", 2: """long""", 3: """yes""", 4: """no"""}
snake_case__ 							=    set("""""".join(["""‘""", """’""", """´""", """`""", """.""", """,""", """-""", """\""""]))




def     snake_case__     (			lowerCamelCase__					: List[str]      ,		lowerCamelCase__					: Tuple=0      ,		lowerCamelCase__					: Tuple=None      ) -> Optional[Any]:
					return [" ".join(x.split(''' '''      )[begin:end]      ) for x in answers if len(x.split(''' '''      )      ) > 1]




def     snake_case__     (			lowerCamelCase__					: Any      ,		lowerCamelCase__					: Any=False      ) -> List[Any]:
					if make_sub_answers:
										# if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
										# *e.g.* if the correct answer contains a prefix such as "the", or "a"
										A_			:  Union[str, Any]			=				(
										    given_answers + get_sub_answers(lowerCamelCase__      ,		begin=1      ) + get_sub_answers(lowerCamelCase__      ,		end=-1      )
										)
					A_			:  List[Any]			=				[]
					for answer in given_answers:
										A_			:  Dict			=				answer.replace('''_'''      ,		''' '''      ).lower()
										A_			:  str			=				''''''.join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else ''' ''' for c in alias      )
										answers.append(''' '''.join(alias.split()      ).strip()      )
					return set(lowerCamelCase__      )




def     snake_case__     (			lowerCamelCase__					: Optional[Any]      ,		lowerCamelCase__					: Dict      ,		lowerCamelCase__					: Union[str, Any]=1      ,		lowerCamelCase__					: Any=1_0_0      ) -> Any:
					A_			,A_			:  str			=				jax.lax.top_k(lowerCamelCase__      ,		lowerCamelCase__      )
					A_			,A_			:  str			=				jax.lax.top_k(lowerCamelCase__      ,		lowerCamelCase__      )

					A_			:  List[Any]			=				best_end_idx[:, None] - best_start_idx[None, :]
					A_			:  Optional[Any]			=				jnp.logical_or(widths < 0      ,		widths > max_size      )
					A_			:  Optional[int]			=				(best_end_scores[:, None] + best_start_scores[None, :]) - (1e8 * mask)
					A_			:  str			=				jnp.argmax(lowerCamelCase__      ).item()

					return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]




def     snake_case__     (			lowerCamelCase__					: List[str]      ) -> Union[str, Any]:
					A_			:  str			=				sample['''question''']['''text''']
					A_			:  Tuple			=				sample['''document''']['''tokens''']['''token''']
					A_			:  List[Any]			=				sample['''document''']['''tokens''']['''is_html''']
					A_			:  Tuple			=				sample['''annotations''']['''long_answer''']
					A_			:  List[Any]			=				sample['''annotations''']['''short_answers''']

					A_			:  Optional[Any]			=				''' '''.join([context[i] for i in range(len(lowerCamelCase__      )      ) if not is_html[i]]      )

					# 0 - No ; 1 - Yes
					for answer in sample["annotations"]["yes_no_answer"]:
										if answer == 0 or answer == 1:
															return {
															    "question": question,
															    "context": context_string,
															    "short": [],
															    "long": [],
															    "category": "no" if answer == 0 else "yes",
															}

					A_			:  List[Any]			=				[]
					for s in short_answers:
										short_targets.extend(s['''text''']      )
					A_			:  Tuple			=				list(set(lowerCamelCase__      )      )

					A_			:  Tuple			=				[]
					for s in long_answers:
										if s["start_token"] == -1:
															continue
										A_			:  Dict			=				context[s['''start_token'''] : s['''end_token''']]
										A_			:  Tuple			=				is_html[s['''start_token'''] : s['''end_token''']]
										A_			:  Optional[Any]			=				''' '''.join([answer[i] for i in range(len(lowerCamelCase__      )      ) if not html[i]]      )
										if new_answer not in long_targets:
															long_targets.append(lowerCamelCase__      )

					A_			:  str			=				'''long_short''' if len(short_targets + long_targets      ) > 0 else '''null'''

					return {
					    "question": question,
					    "context": context_string,
					    "short": short_targets,
					    "long": long_targets,
					    "category": category,
					}




def     snake_case__     (			) -> List[Any]:
					A_			:  int			=				load_from_disk('''natural-questions-validation'''      )
					A_			:  Optional[int]			=				dataset.map(lowerCamelCase__      ).remove_columns(['''annotations''', '''document''', '''id''']      )
					print(lowerCamelCase__      )

					A_			:  Tuple			=				dataset.filter(lambda lowerCamelCase__      : (len(x['''question''']      ) + len(x['''context''']      )) < 4 * 4_0_9_6      )
					A_			:  Optional[Any]			=				short_validation_dataset.filter(lambda lowerCamelCase__      : x["category"] != "null"      )
					short_validation_dataset

					A_			:  Union[str, Any]			=				'''vasudevgupta/flax-bigbird-natural-questions'''
					A_			:  List[str]			=				FlaxBigBirdForNaturalQuestions.from_pretrained(lowerCamelCase__      )
					A_			:  int			=				BigBirdTokenizerFast.from_pretrained(lowerCamelCase__      )

					@jax.jit
					def forward(*lowerCamelCase__					: Any      ,		**lowerCamelCase__					: Optional[Any]      ):
										A_			,A_			,A_			:  Tuple			=				model(*lowerCamelCase__      ,		**lowerCamelCase__      )
										return start_logits, end_logits, jnp.argmax(lowerCamelCase__      ,		axis=-1      )

					def evaluate(lowerCamelCase__					: List[str]      ):
										# encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
										A_			:  List[Any]			=				tokenizer(
										    example['''question''']      ,		example['''context''']      ,		return_tensors='''np'''      ,		max_length=4_0_9_6      ,		padding='''max_length'''      ,		truncation=lowerCamelCase__      ,		)

										A_			,A_			,A_			:  Optional[Any]			=				forward(**lowerCamelCase__      )

										A_			:  Optional[Any]			=				CATEGORY_MAPPING[category.item()]

										A_			:  str			=				example['''long'''] + example['''short''']
										if example["category"] in ["yes", "no", "null"]:
															A_			:  str			=				[example['''category''']]
										A_			:  Optional[Any]			=				example['''category'''] != '''null'''
										# Now target can be: "yes", "no", "null", "list of long & short answers"

										if predicted_category in ["yes", "no", "null"]:
															A_			:  int			=				[predicted_category]
															A_			:  Optional[int]			=				example['''output'''] == example['''targets''']
															A_			:  Dict			=				predicted_category != '''null'''
															return example

										A_			:  List[Any]			=				3_8 if predicted_category == '''short''' else 1_0_2_4
										A_			,A_			:  str			=				get_best_valid_start_end_idx(
										    start_scores[0]      ,		end_scores[0]      ,		top_k=8      ,		max_size=lowerCamelCase__      )

										A_			:  List[Any]			=				inputs['''input_ids'''][0].tolist()
										A_			:  Dict			=				[tokenizer.decode(input_ids[start_score : end_score + 1]      )]

										A_			:  Optional[int]			=				expand_to_aliases(example['''targets''']      ,		make_sub_answers=lowerCamelCase__      )
										A_			:  str			=				expand_to_aliases(example['''output''']      )

										# some preprocessing to both prediction and answer
										A_			:  Tuple			=				{''''''.join(a.split()      ) for a in answers}
										A_			:  Optional[int]			=				{''''''.join(p.split()      ) for p in predictions}
										A_			:  List[str]			=				{s for s in predictions if s not in ['''``''', '''\'\'''', '''`''', '''\'''']}

										# if there is a common element, it's a exact match
										A_			:  List[Any]			=				len(list(answers & predictions      )      ) > 0
										A_			:  Dict			=				predicted_category != '''null''' and len(lowerCamelCase__      ) > 0

										return example

					A_			:  Dict			=				short_validation_dataset.map(lowerCamelCase__      )

					A_			:  List[str]			=				len(lowerCamelCase__      )
					A_			:  Any			=				len(short_validation_dataset.filter(lambda lowerCamelCase__      : x["match"] == 1      )      )
					print('''EM score:'''      ,		(matched / total) * 1_0_0      ,		'''%'''      )


if __name__ == "__main__":
							main()



