
"""simple docstring"""




from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

__lowerCAmelCase							:  Optional[int] 				=		{
    name: (getattr(transformers, name), getattr(transformers, name + '''Fast''')) for name in SLOW_TO_FAST_CONVERTERS
}

__lowerCAmelCase							:  Optional[Any] 				=		datasets.load_dataset('''xnli''', split='''test+validation''')

__lowerCAmelCase							:  Optional[int] 				=		0
__lowerCAmelCase							:  Tuple 				=		0
__lowerCAmelCase							:  List[str] 				=		0
__lowerCAmelCase							:  Optional[int] 				=		0

def __snake_case (      UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase    )       ->    Union[str, Any]:


		"""simple docstring"""





		if spm_diff == list(reversed(UpperCamelCase    )    ):
				# AAA -> AA+A vs A+AA case.
				return True
		elif len(UpperCamelCase    ) == len(UpperCamelCase    ) and fast.decode(UpperCamelCase    ) == fast.decode(UpperCamelCase    ):
				# Second order OK
				# Barrich -> Barr + ich vs Bar + rich
				return True
		a__   		=     slow.encode(slow.decode(UpperCamelCase    )    )
		a__   		=     fast.encode(fast.decode(UpperCamelCase    )    )
		if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
				# Type 3 error.
				# Snehagatha ->
				#       Sne, h, aga, th, a
				#       Sne, ha, gat, ha
				# Encoding the wrong with sp does not even recover what spm gave us
				# It fits tokenizer however...
				return True
		return False

def __snake_case (      UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase    )       ->    List[str]:


		"""simple docstring"""





		a__   		=     fast.encode_plus(UpperCamelCase    )[0]
		a__   		=     enc.offsets
		a__		,	a__   		=     offsets[idx], offsets[idx - 1]
		if curr is not None and line[curr[0] : curr[1]] == "\u200f":
				return True
		if prev is not None and line[prev[0] : prev[1]] == "\u200f":
				return True

def __snake_case (      UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase    )       ->    Optional[int]:


		"""simple docstring"""





		for i, (spm_id, tok_id) in enumerate(zip(UpperCamelCase						,					UpperCamelCase    )    ):
				if spm_id != tok_id:
						break
		a__   		=     i
		for i, (spm_id, tok_id) in enumerate(zip(reversed(UpperCamelCase    )						,					reversed(UpperCamelCase    )    )    ):
				if spm_id != tok_id:
						break
		a__   		=     len(UpperCamelCase    ) - i

		a__   		=     spm_ids[first:last]
		a__   		=     tok_ids[first:last]

		if check_diff(UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase    ):
				return True

		if check_LTR_mark(UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase    ):
				return True

		if last - first > 5:
				# We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
				a__   		=     Counter(spm_ids[first:last]    )
				a__   		=     Counter(tok_ids[first:last]    )

				a__   		=     {spm_ for (spm_, si) in spms.items() if toks.get(spm_						,					0    ) == si}
				a__   		=     3
				for i in range(last - first - min_width    ):
						if all(spm_ids[first + i + j] in removable_tokens for j in range(UpperCamelCase    )    ):
								a__   		=     [
								    k
								    for k in range(last - first - min_width    )
								    if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
								]
								for j in possible_matches:
										if check_diff(spm_ids[first : first + i]						,					tok_ids[first : first + j]						,					UpperCamelCase						,					UpperCamelCase    ) and check_details(
										    UpperCamelCase						,					spm_ids[first + i : last]						,					tok_ids[first + j : last]						,					UpperCamelCase						,					UpperCamelCase						,					):
												return True

		print(f"Spm: {[fast.decode([spm_ids[i]]    ) for i in range(UpperCamelCase						,					UpperCamelCase    )]}"    )
		try:
				print(f"Tok: {[fast.decode([tok_ids[i]]    ) for i in range(UpperCamelCase						,					UpperCamelCase    )]}"    )
		except Exception:
				pass

		a__   		=     fast.decode(spm_ids[:first]    )
		a__   		=     fast.decode(spm_ids[last:]    )
		a__   		=     fast.decode(spm_ids[first:last]    )
		print()
		print(UpperCamelCase    )
		return False

def __snake_case (      UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase    )       ->    List[Any]:


		"""simple docstring"""





		global perfect
		global imperfect
		global wrong
		global total

		a__   		=     slow.encode(UpperCamelCase    )
		a__   		=     fast.encode(UpperCamelCase    )

		a__   		=     False
		total += 1

		if slow_ids != fast_ids:
				if check_details(UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase    ):
						a__   		=     True
						imperfect += 1
				else:
						wrong += 1
		else:
				perfect += 1

		if total % 10_000 == 0:
				print(f"({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})"    )

		if skip_assert:
				return

		assert (
		    slow_ids == fast_ids
		), f"line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(UpperCamelCase    )}\n{fast.tokenize(UpperCamelCase    )}"

def __snake_case (      UpperCamelCase						,					UpperCamelCase    )       ->    List[Any]:


		"""simple docstring"""





		global batch_total
		for i in range(len(UpperCamelCase    )    ):
				# premise, all languages
				for text in dataset[i]["premise"].values():
						test_string(UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase    )

				# hypothesis, all languages
				for text in dataset[i]["hypothesis"]["translation"]:
						test_string(UpperCamelCase						,					UpperCamelCase						,					UpperCamelCase    )


if __name__ == "__main__":
					for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
										__lowerCAmelCase							:  List[Any] 				=		list(slow_class.max_model_input_sizes.keys())
										for checkpoint in checkpoint_names:
															__lowerCAmelCase							:  Tuple 				=		0
															__lowerCAmelCase							:  Dict 				=		0
															__lowerCAmelCase							:  Any 				=		0
															__lowerCAmelCase							:  Dict 				=		0

															print(f"========================== Checking {name}: {checkpoint} ==========================")
															__lowerCAmelCase							:  Dict 				=		slow_class.from_pretrained(checkpoint, force_download=True)
															__lowerCAmelCase							:  List[Any] 				=		fast_class.from_pretrained(checkpoint, force_download=True)
															test_tokenizer(slow, fast)
															print(f"Accuracy {perfect * 100 / total:.2f}")






"""simple docstring"""




from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy
class SCREAMING_SNAKE_CASE       (      lowerCAmelCase					):






	'''simple docstring'''







	snake_case__							:		Optional[Any]         =		'MarkupLMFeatureExtractor'
	snake_case__							:		Tuple         =		('MarkupLMTokenizer', 'MarkupLMTokenizerFast')
	snake_case__							:		Dict         =		True


	def __call__(  self :Any       ,    __magic_name__ :Dict=None       ,    __magic_name__ :Dict=None       ,    __magic_name__ :List[str]=None       ,    __magic_name__ :List[str]=None       ,    __magic_name__ :Tuple=None       ,    __magic_name__ :bool = True       ,    __magic_name__ :Union[bool, str, PaddingStrategy] = False       ,    __magic_name__ :Union[bool, str, TruncationStrategy] = None       ,    __magic_name__ :Optional[int] = None       ,    __magic_name__ :int = 0       ,    __magic_name__ :Optional[int] = None       ,    __magic_name__ :Optional[bool] = None       ,    __magic_name__ :Optional[bool] = None       ,    __magic_name__ :bool = False       ,    __magic_name__ :bool = False       ,    __magic_name__ :bool = False       ,    __magic_name__ :bool = False       ,    __magic_name__ :bool = True       ,    __magic_name__ :Optional[Union[str, TensorType]] = None       ,    **__magic_name__ :Optional[Any]       ,    )			->   BatchEncoding:

			'''simple docstring'''






			if self.parse_html:
					if html_strings is None:
							raise ValueError('''Make sure to pass HTML strings in case `parse_html` is set to `True`'''			)

					if nodes is not None or xpaths is not None or node_labels is not None:
							raise ValueError(
							    '''Please don\'t pass nodes, xpaths nor node labels in case `parse_html` is set to `True`'''			)

					a__   		=     self.feature_extractor(__magic_name__			)
					a__   		=     features['''nodes''']
					a__   		=     features['''xpaths''']
			else:
					if html_strings is not None:
							raise ValueError('''You have passed HTML strings but `parse_html` is set to `False`.'''			)
					if nodes is None or xpaths is None:
							raise ValueError('''Make sure to pass nodes and xpaths in case `parse_html` is set to `False`'''			)

        # # second, apply the tokenizer
			if questions is not None and self.parse_html:
					if isinstance(__magic_name__       ,    __magic_name__			):
							a__   		=     [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

			a__   		=     self.tokenizer(
			    text=questions if questions is not None else nodes       ,    text_pair=nodes if questions is not None else None       ,    xpaths=__magic_name__       ,    node_labels=__magic_name__       ,    add_special_tokens=__magic_name__       ,    padding=__magic_name__       ,    truncation=__magic_name__       ,    max_length=__magic_name__       ,    stride=__magic_name__       ,    pad_to_multiple_of=__magic_name__       ,    return_token_type_ids=__magic_name__       ,    return_attention_mask=__magic_name__       ,    return_overflowing_tokens=__magic_name__       ,    return_special_tokens_mask=__magic_name__       ,    return_offsets_mapping=__magic_name__       ,    return_length=__magic_name__       ,    verbose=__magic_name__       ,    return_tensors=__magic_name__       ,    **__magic_name__       ,    )

			return encoded_inputs


	def    _UpperCamelCase							(  self :Tuple       ,    *__magic_name__ :Optional[int]       ,    **__magic_name__ :Any			)			->   int:

			'''simple docstring'''






			return self.tokenizer.batch_decode(*__magic_name__       ,    **__magic_name__			)


	def    _UpperCamelCase							(  self :Tuple       ,    *__magic_name__ :Union[str, Any]       ,    **__magic_name__ :Union[str, Any]			)			->   Dict:

			'''simple docstring'''






			return self.tokenizer.decode(*__magic_name__       ,    **__magic_name__			)






	@property
	def    _UpperCamelCase							(  self :Optional[int]			)			->   Union[str, Any]:

			'''simple docstring'''






			a__   		=     self.tokenizer.model_input_names
			return tokenizer_input_names




