






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


_a   			=						logging.get_logger(__name__)

_a   			=						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_a   			=						{
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

_a   			=						{
    '''microsoft/mpnet-base''': 512,
}

_a   			=						{
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}





class __lowerCamelCase				(   snake_case__):





	"""simple docstring"""






	UpperCamelCase__			=   VOCAB_FILES_NAMES
	UpperCamelCase__			=   PRETRAINED_VOCAB_FILES_MAP
	UpperCamelCase__			=   PRETRAINED_INIT_CONFIGURATION
	UpperCamelCase__			=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCamelCase__			=   MPNetTokenizer
	UpperCamelCase__			=   ["input_ids", "attention_mask"]

	def __init__(							self ,					UpperCAmelCase=None ,					UpperCAmelCase=None ,					UpperCAmelCase=True ,					UpperCAmelCase="<s>" ,					UpperCAmelCase="</s>" ,					UpperCAmelCase="</s>" ,					UpperCAmelCase="<s>" ,					UpperCAmelCase="[UNK]" ,					UpperCAmelCase="<pad>" ,					UpperCAmelCase="<mask>" ,					UpperCAmelCase=True ,					UpperCAmelCase=None ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							super().__init__(
							    UpperCAmelCase ,					tokenizer_file=UpperCAmelCase ,					do_lower_case=UpperCAmelCase ,					bos_token=UpperCAmelCase ,					eos_token=UpperCAmelCase ,					sep_token=UpperCAmelCase ,					cls_token=UpperCAmelCase ,					unk_token=UpperCAmelCase ,					pad_token=UpperCAmelCase ,					mask_token=UpperCAmelCase ,					tokenize_chinese_chars=UpperCAmelCase ,					strip_accents=UpperCAmelCase ,					**UpperCAmelCase ,					)

							_UpperCAmelCase             =    json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
							if (
							    pre_tok_state.get('lowercase' ,					UpperCAmelCase					) != do_lower_case
							    or pre_tok_state.get('strip_accents' ,					UpperCAmelCase					) != strip_accents
							):
													_UpperCAmelCase             =    getattr(UpperCAmelCase ,					pre_tok_state.pop('type'					)					)
													_UpperCAmelCase             =    do_lower_case
													_UpperCAmelCase             =    strip_accents
													_UpperCAmelCase             =    pre_tok_class(**UpperCAmelCase					)

							_UpperCAmelCase             =    do_lower_case

	@property
	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							if self._mask_token is None:
													if self.verbose:
																			logger.error('Using mask_token, but it is not set yet.'					)
													return None
							return str(self._mask_token					)

	@mask_token.setter
	def  UpperCamelCase					(							self ,					UpperCAmelCase					):
							"""simple docstring"""







							_UpperCAmelCase             =    AddedToken(UpperCAmelCase ,					lstrip=UpperCAmelCase ,					rstrip=UpperCAmelCase					) if isinstance(UpperCAmelCase ,					UpperCAmelCase					) else value
							_UpperCAmelCase             =    value

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase=None					):
							"""simple docstring"""







							_UpperCAmelCase             =    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
													return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase = None					):
							"""simple docstring"""







							_UpperCAmelCase             =    [self.sep_token_id]
							_UpperCAmelCase             =    [self.cls_token_id]

							if token_ids_a is None:
													return len(cls + token_ids_a + sep					) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase = None					):
							"""simple docstring"""







							_UpperCAmelCase             =    self._tokenizer.model.save(UpperCAmelCase ,					name=UpperCAmelCase					)
							return tuple(UpperCAmelCase					)








import dataclasses
import enum
import functools
import math
import re

# The following script is adapted from the script of TaPas.
# Original: https://github.com/google-research/tapas/master/wikisql_utils.py
from typing import Any, List, Text


_a   			=						'''none'''
_a   			=						'''none'''


def __A					(   __lowerCAmelCase				,	__lowerCAmelCase			)->						Any:





						"""simple docstring"""



						_UpperCAmelCase             =    value.split(__lowerCAmelCase			)
						return len(__lowerCAmelCase			) > 1 and any((len(__lowerCAmelCase			) == 3 for x in split)			)


def __A					(   __lowerCAmelCase			)->						List[str]:





						"""simple docstring"""



						if isinstance(__lowerCAmelCase				,	__lowerCAmelCase			):
												return value
						if isinstance(__lowerCAmelCase				,	__lowerCAmelCase			):
												return float(__lowerCAmelCase			)
						if not isinstance(__lowerCAmelCase				,	__lowerCAmelCase			):
												raise ValueError('Argument value is not a string. Can\'t parse it as float'			)
						_UpperCAmelCase             =    value

						try:
												# Example: 1,000.7
												if "." in sanitized and "," in sanitized:
																		return float(sanitized.replace(','				,	''			)			)
												# 1,000
												if "," in sanitized and _split_thousands(','				,	__lowerCAmelCase			):
																		return float(sanitized.replace(','				,	''			)			)
												# 5,5556
												if "," in sanitized and sanitized.count(','			) == 1 and not _split_thousands(','				,	__lowerCAmelCase			):
																		return float(sanitized.replace(','				,	'.'			)			)
												# 0.0.0.1
												if sanitized.count('.'			) > 1:
																		return float(sanitized.replace('.'				,	''			)			)
												# 0,0,0,1
												if sanitized.count(','			) > 1:
																		return float(sanitized.replace(','				,	''			)			)
												return float(__lowerCAmelCase			)
						except ValueError:
												# Avoid adding the sanitized value in the error message.
												raise ValueError('Unable to convert value to float'			)


def __A					(   __lowerCAmelCase			)->						Any:





						"""simple docstring"""



						if answer is None:
												return None
						try:
												_UpperCAmelCase             =    convert_to_float(__lowerCAmelCase			)
												if isinstance(__lowerCAmelCase				,	__lowerCAmelCase			) and math.isnan(__lowerCAmelCase			):
																		return None
												return value
						except ValueError:
												return answer.lower()


_a   			=						{
    '''text''': lambda x: x,
    '''real''': convert_to_float,
}





class __lowerCamelCase				(   enum.Enum):





	"""simple docstring"""






	UpperCamelCase__			=   0
	UpperCamelCase__			=   1
	UpperCamelCase__			=   2
	UpperCamelCase__			=   3
	UpperCamelCase__			=   4
	UpperCamelCase__			=   5





class __lowerCamelCase				(   enum.Enum):





	"""simple docstring"""






	UpperCamelCase__			=   0
	UpperCamelCase__			=   1
	UpperCamelCase__			=   2





@dataclasses.dataclass
class __lowerCamelCase				:





	"""simple docstring"""






	UpperCamelCase__			=   42
	UpperCamelCase__			=   42
	UpperCamelCase__			=   42



_a   			=						re.compile(r'''\w+|[^\w\s]+''', re.UNICODE | re.MULTILINE | re.DOTALL)


def __A					(   __lowerCAmelCase			)->						Optional[Any]:





						"""simple docstring"""



						return list(_TOKENIZER.findall(x.lower()			)			)


def __A					(   __lowerCAmelCase				,	__lowerCAmelCase				,	__lowerCAmelCase			)->						Any:





						"""simple docstring"""



						if operator == _Operator.EQUALS:
												return src == tgt
						elif operator == _Operator.GREATER:
												return src > tgt
						elif operator == _Operator.LESSER:
												return src < tgt
						raise ValueError(F"""Unknown operator: {operator}"""			)


def __A					(   __lowerCAmelCase				,	__lowerCAmelCase				,	__lowerCAmelCase			)->						str:





						"""simple docstring"""



						_UpperCAmelCase             =    table['types']
						return _TYPE_CONVERTER[types[column]](__lowerCAmelCase			)


def __A					(   __lowerCAmelCase			)->						str:





						"""simple docstring"""



						return isinstance(__lowerCAmelCase				,	__lowerCAmelCase			)


def __A					(   __lowerCAmelCase				,	__lowerCAmelCase				,	__lowerCAmelCase			)->						str:





						"""simple docstring"""



						for cond in conditions:
												_UpperCAmelCase             =    row[cond.column]

												_UpperCAmelCase             =    _parse_value(__lowerCAmelCase				,	cond.column				,	cond.cmp_value			)

												if _is_string(__lowerCAmelCase			) and _is_string(__lowerCAmelCase			):
																		_UpperCAmelCase             =    _normalize_for_match(__lowerCAmelCase			)
																		_UpperCAmelCase             =    _normalize_for_match(__lowerCAmelCase			)

												if not isinstance(__lowerCAmelCase				,	type(__lowerCAmelCase			)			):
																		raise ValueError('Type difference {} != {}'.format(type(__lowerCAmelCase			)				,	type(__lowerCAmelCase			)			)			)

												if not _compare(cond.operator				,	__lowerCAmelCase				,	__lowerCAmelCase			):
																		return False
						return True


def __A					(   __lowerCAmelCase				,	__lowerCAmelCase				,	__lowerCAmelCase			)->						str:





						"""simple docstring"""



						if not answer_coordinates:
												if aggregation_op == _Aggregation.COUNT:
																		return 0.0
												else:
																		return EMPTY_ANSWER_AGG

    # Count can support non numeric answers.
						if aggregation_op == _Aggregation.COUNT:
												return float(len(__lowerCAmelCase			)			)

						# If we have just one answer, if float returns it or try a conversion.
						_UpperCAmelCase             =    [table['rows'][i][j] for (i, j) in answer_coordinates]
						if len(__lowerCAmelCase			) == 1:
												try:
																		return convert_to_float(values[0]			)
												except ValueError as e:
																		if aggregation_op != _Aggregation.NONE:
																								raise e

						if aggregation_op == _Aggregation.NONE:
												return None

						# Other aggregation only support numeric values. Bail out if we have strings.
						if not all((isinstance(__lowerCAmelCase				,	(int, float)			) for v in values)			):
												return None

						if aggregation_op == _Aggregation.SUM:
												return float(sum(__lowerCAmelCase			)			)
						elif aggregation_op == _Aggregation.AVERAGE:
												return sum(__lowerCAmelCase			) / len(__lowerCAmelCase			)
						else:
												raise ValueError(F"""Unknown aggregation: {aggregation_op}"""			)


def __A					(   __lowerCAmelCase				,	__lowerCAmelCase			)->						Any:





						"""simple docstring"""



						_UpperCAmelCase             =    sql_query['agg']
						if aggregation_op_index >= 3:
												_UpperCAmelCase             =    _Aggregation(__lowerCAmelCase			)
						else:
												_UpperCAmelCase             =    _Aggregation.NONE

						_UpperCAmelCase             =    sql_query['sel']
						_UpperCAmelCase             =    [
						    _Condition(__lowerCAmelCase				,	_Operator(__lowerCAmelCase			)				,	__lowerCAmelCase			)
						    for column, operator, cmp_value in zip(
						        sql_query['conds']['column_index']				,	sql_query['conds']['operator_index']				,	sql_query['conds']['condition']			)
						]

						_UpperCAmelCase             =    []
						for row in range(len(table['rows']			)			):
												if _respect_conditions(__lowerCAmelCase				,	table['rows'][row]				,	__lowerCAmelCase			):
																		indices.append((row, target_column)			)

						if not indices:
												return [], aggregation_op

						if len(__lowerCAmelCase			) == 1:
												return indices, aggregation_op

						# Parsing of MIN/MAX.
						if aggregation_op_index in (1, 2):
												_UpperCAmelCase             =    {2: min, 1: max}
												_UpperCAmelCase             =    [(table['rows'][i][j], index) for index, (i, j) in enumerate(__lowerCAmelCase			)]
												_UpperCAmelCase             =    functools.reduce(operators[sql_query['agg']]				,	__lowerCAmelCase			)

												_UpperCAmelCase             =    [indices[reduced[1]]]
												return ret, _Aggregation.NONE

						return indices, aggregation_op


def __A					(   __lowerCAmelCase				,	__lowerCAmelCase				,	__lowerCAmelCase			)->						Dict:





						"""simple docstring"""



						if float_answer is not None:
												return [str(__lowerCAmelCase			)]
						return [str(table['real_rows'][r][c]			) for r, c in answer_coordinates]


def __A					(   __lowerCAmelCase				,	__lowerCAmelCase			)->						List:





						"""simple docstring"""



						_UpperCAmelCase      ,				_UpperCAmelCase             =    _get_answer_coordinates(__lowerCAmelCase				,	__lowerCAmelCase			)
						_UpperCAmelCase             =    _get_float_answer(__lowerCAmelCase				,	__lowerCAmelCase				,	__lowerCAmelCase			)
						_UpperCAmelCase             =    _get_answer_text(__lowerCAmelCase				,	__lowerCAmelCase				,	__lowerCAmelCase			)
						# keep the original data the same with TaPas
						if len(__lowerCAmelCase			) == 0:
												_UpperCAmelCase             =    [EMPTY_ANSWER]
						return answer_text
