
import json
import os
import re
import warnings
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__						:Union[str, Any]					    =       logging.get_logger(__name__)

lowercase__						:Optional[int]					    =       {
    "source_spm": "source.spm",
    "target_spm": "target.spm",
    "vocab": "vocab.json",
    "target_vocab_file": "target_vocab.json",
    "tokenizer_config_file": "tokenizer_config.json",
}

lowercase__						:int					    =       {
    "source_spm": {
        "Helsinki-NLP/opus-mt-en-de": "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/source.spm"
    },
    "target_spm": {
        "Helsinki-NLP/opus-mt-en-de": "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/target.spm"
    },
    "vocab": {
        "Helsinki-NLP/opus-mt-en-de": "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json"
    },
    "tokenizer_config_file": {
        "Helsinki-NLP/opus-mt-en-de": (
            "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/tokenizer_config.json"
        )
    },
}

lowercase__						:Optional[int]					    =       {"Helsinki-NLP/opus-mt-en-de": 512}
lowercase__						:str					    =       {}







class 							lowercase						(   _a						):
							lowercase_			:			Tuple			   =VOCAB_FILES_NAMES
							lowercase_			:			Tuple			   =PRETRAINED_VOCAB_FILES_MAP
							lowercase_			:			List[str]			   =PRETRAINED_INIT_CONFIGURATION
							lowercase_			:			Any			   =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_			:			List[str]			   =['''input_ids''', '''attention_mask''']
							lowercase_			:			Optional[int]			   =re.compile('''>>.+<<'''						)  # type: re.Pattern




							def __init__(							self  ,A__  ,A__  ,A__  ,A__=None  ,A__=None  ,A__=None  ,A__="<unk>"  ,A__="</s>"  ,A__="<pad>"  ,A__=5_1_2  ,A__ = None  ,A__=False  ,**A__  ,):
												lowercase		      =		{} if sp_model_kwargs is None else sp_model_kwargs

												super().__init__(
												    # bos_token=bos_token,  unused. Start decoding with config.decoder_start_token_id
												    source_lang=_SCREAMING_SNAKE_CASE  ,target_lang=_SCREAMING_SNAKE_CASE  ,unk_token=_SCREAMING_SNAKE_CASE  ,eos_token=_SCREAMING_SNAKE_CASE  ,pad_token=_SCREAMING_SNAKE_CASE  ,model_max_length=_SCREAMING_SNAKE_CASE  ,sp_model_kwargs=self.sp_model_kwargs  ,target_vocab_file=_SCREAMING_SNAKE_CASE  ,separate_vocabs=_SCREAMING_SNAKE_CASE  ,**_SCREAMING_SNAKE_CASE  ,)
												assert Path(_SCREAMING_SNAKE_CASE).exists(), f'cannot find spm source {source_spm}'

												lowercase		      =		separate_vocabs
												lowercase		      =		load_json(_SCREAMING_SNAKE_CASE)
												if self.unk_token not in self.encoder:
																	raise KeyError('''<unk> token must be in vocab''')
												assert self.pad_token in self.encoder

												if separate_vocabs:
																	lowercase		      =		load_json(_SCREAMING_SNAKE_CASE)
																	lowercase		      =		{v: k for k, v in self.target_encoder.items()}
																	lowercase		      =		[]
												else:
																	lowercase		      =		{v: k for k, v in self.encoder.items()}
																	lowercase		      =		[k for k in self.encoder if k.startswith('''>>''') and k.endswith('''<<''')]

												lowercase		      =		source_lang
												lowercase		      =		target_lang
												lowercase		      =		[source_spm, target_spm]

												# load SentencePiece model for pre-processing
												lowercase		      =		load_spm(_SCREAMING_SNAKE_CASE  ,self.sp_model_kwargs)
												lowercase		      =		load_spm(_SCREAMING_SNAKE_CASE  ,self.sp_model_kwargs)
												lowercase		      =		self.spm_source
												lowercase		      =		self.encoder

												# Multilingual target side: default to using first supported language code.

												self._setup_normalizer()




							def 					A__ (							self):






												try:
																	from sacremoses import MosesPunctNormalizer

																	lowercase		      =		MosesPunctNormalizer(self.source_lang).normalize
												except (ImportError, FileNotFoundError):
																	warnings.warn('''Recommended: pip install sacremoses.''')
																	lowercase		      =		lambda A__: x




							def 					A__ (							self  ,A__):
												return self.punc_normalizer(_SCREAMING_SNAKE_CASE) if x else ""




							def 					A__ (							self  ,A__):
												return self.current_encoder.get(_SCREAMING_SNAKE_CASE  ,self.current_encoder[self.unk_token])




							def 					A__ (							self  ,A__):
												lowercase		      =		self.language_code_re.match(_SCREAMING_SNAKE_CASE)
												lowercase		      =		[match.group(0)] if match else []
												return code, self.language_code_re.sub(''''''  ,_SCREAMING_SNAKE_CASE)




							def 					A__ (							self  ,A__):
												lowercase		      =		self.remove_language_code(_SCREAMING_SNAKE_CASE)
												lowercase		      =		self.current_spm.encode(_SCREAMING_SNAKE_CASE  ,out_type=_SCREAMING_SNAKE_CASE)
												return code + pieces




							def 					A__ (							self  ,A__):
												return self.decoder.get(_SCREAMING_SNAKE_CASE  ,self.unk_token)




							def 					A__ (							self  ,A__  ,**A__):
												return super().batch_decode(_SCREAMING_SNAKE_CASE  ,**_SCREAMING_SNAKE_CASE)




							def 					A__ (							self  ,A__  ,**A__):
												return super().decode(_SCREAMING_SNAKE_CASE  ,**_SCREAMING_SNAKE_CASE)




							def 					A__ (							self  ,A__):
												lowercase		      =		self.spm_source if self._decode_use_source_tokenizer else self.spm_target
												lowercase		      =		[]
												lowercase		      =		""""""
												for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																						out_string += sp_model.decode_pieces(_SCREAMING_SNAKE_CASE) + token + " "
																						lowercase		      =		[]
																	else:
																						current_sub_tokens.append(_SCREAMING_SNAKE_CASE)
												out_string += sp_model.decode_pieces(_SCREAMING_SNAKE_CASE)
												return out_string.strip()




							def 					A__ (							self  ,A__  ,A__=None):






												if token_ids_a is None:
																	return token_ids_a + [self.eos_token_id]
												# We don't expect to process pairs, but leave the pair logic for API consistency
												return token_ids_a + token_ids_a + [self.eos_token_id]




							def 					A__ (							self):
												lowercase		      =		self.spm_source
												lowercase		      =		self.encoder




							def 					A__ (							self):
												lowercase		      =		self.spm_target
												if self.separate_vocabs:
																	lowercase		      =		self.target_encoder




							@property
							def 					A__ (							self):
												return len(self.encoder)




							def 					A__ (							self  ,A__  ,A__ = None):






												if not os.path.isdir(_SCREAMING_SNAKE_CASE):
																	logger.error(f'Vocabulary path ({save_directory}) should be a directory')
																	return
												lowercase		      =		[]

												if self.separate_vocabs:
																	lowercase		      =		os.path.join(
																	    _SCREAMING_SNAKE_CASE  ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab''']  ,)
																	lowercase		      =		os.path.join(
																	    _SCREAMING_SNAKE_CASE  ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''target_vocab_file''']  ,)
																	save_json(self.encoder  ,_SCREAMING_SNAKE_CASE)
																	save_json(self.target_encoder  ,_SCREAMING_SNAKE_CASE)
																	saved_files.append(_SCREAMING_SNAKE_CASE)
																	saved_files.append(_SCREAMING_SNAKE_CASE)
												else:
																	lowercase		      =		os.path.join(
																	    _SCREAMING_SNAKE_CASE  ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab'''])
																	save_json(self.encoder  ,_SCREAMING_SNAKE_CASE)
																	saved_files.append(_SCREAMING_SNAKE_CASE)

												for spm_save_filename, spm_orig_path, spm_model in zip(
												    [VOCAB_FILES_NAMES['''source_spm'''], VOCAB_FILES_NAMES['''target_spm''']]  ,self.spm_files  ,[self.spm_source, self.spm_target]  ,):
																	lowercase		      =		os.path.join(
																	    _SCREAMING_SNAKE_CASE  ,(filename_prefix + '''-''' if filename_prefix else '''''') + spm_save_filename)
																	if os.path.abspath(_SCREAMING_SNAKE_CASE) != os.path.abspath(_SCREAMING_SNAKE_CASE) and os.path.isfile(_SCREAMING_SNAKE_CASE):
																						copyfile(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE)
																						saved_files.append(_SCREAMING_SNAKE_CASE)
																	elif not os.path.isfile(_SCREAMING_SNAKE_CASE):
																						with open(_SCREAMING_SNAKE_CASE  ,'''wb''') as fi:
																											lowercase		      =		spm_model.serialized_model_proto()
																											fi.write(_SCREAMING_SNAKE_CASE)
																						saved_files.append(_SCREAMING_SNAKE_CASE)

												return tuple(_SCREAMING_SNAKE_CASE)




							def 					A__ (							self):
												return self.get_src_vocab()




							def 					A__ (							self):
												return dict(self.encoder  ,**self.added_tokens_encoder)




							def 					A__ (							self):
												return dict(self.target_encoder  ,**self.added_tokens_decoder)




							def __getstate__(							self):
												lowercase		      =		self.__dict__.copy()
												state.update(
												    {k: None for k in ['''spm_source''', '''spm_target''', '''current_spm''', '''punc_normalizer''', '''target_vocab_file''']})
												return state




							def __setstate__(							self  ,A__):
												lowercase		      =		d

												# for backward compatibility
												if not hasattr(self  ,'''sp_model_kwargs'''):
																	lowercase		      =		{}

												lowercase		      =		(load_spm(_SCREAMING_SNAKE_CASE  ,self.sp_model_kwargs) for f in self.spm_files)
												lowercase		      =		self.spm_source
												self._setup_normalizer()




							def 					A__ (							self  ,*A__  ,**A__):
												return 1




							def 					A__ (							self  ,A__):
												lowercase		      =		set(self.all_special_ids)  # call it once instead of inside list comp
												all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special
												return [1 if x in all_special_ids else 0 for x in seq]





							def 					A__ (							self  ,A__  ,A__ = None  ,A__ = False):






												if already_has_special_tokens:
																	return self._special_token_mask(_SCREAMING_SNAKE_CASE)
												elif token_ids_a is None:
																	return self._special_token_mask(_SCREAMING_SNAKE_CASE) + [1]
												else:
																	return self._special_token_mask(token_ids_a + token_ids_a) + [1]



def 			UpperCamelCase       (				lowerCAmelCase__					,     lowerCAmelCase__						):
					'''simple docstring'''
					lowercase		      =		sentencepiece.SentencePieceProcessor(**UpperCAmelCase__						)
					spm.Load(UpperCAmelCase__						)
					return spm



def 			UpperCamelCase       (				lowerCAmelCase__					,     lowerCAmelCase__						):
					'''simple docstring'''
					with open(UpperCAmelCase__					,     '''w'''						) as f:
										json.dump(UpperCAmelCase__					,     UpperCAmelCase__					,     indent=2						)



def 			UpperCamelCase       (				lowerCAmelCase__						):
					'''simple docstring'''
					with open(UpperCAmelCase__					,     '''r'''						) as f:
										return json.load(UpperCAmelCase__						)






'''simple docstring'''


import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class     __snake_case			(    _a					,			unittest.TestCase	):
		a_          =					RealmTokenizer
		a_          =					RealmTokenizerFast
		a_          =					True
		a_          =					True
		a_          =					filter_non_english




		def 	_snake_case	(  self					:Union[str, Any])					->			List[Any]:






								'''simple docstring'''






								super().setUp()

								UpperCamelCase_     :	Optional[int]					     =     [
								    """[UNK]""",
								    """[CLS]""",
								    """[SEP]""",
								    """[PAD]""",
								    """[MASK]""",
								    """want""",
								    """##want""",
								    """##ed""",
								    """wa""",
								    """un""",
								    """runn""",
								    """##ing""",
								    """,""",
								    """low""",
								    """lowest""",
								]
								UpperCamelCase_     :	Tuple					     =     os.path.join(self.tmpdirname			,    VOCAB_FILES_NAMES["""vocab_file"""])
								with open(self.vocab_file			,    """w"""			,    encoding="""utf-8""") as vocab_writer:
														vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))




		def 	_snake_case	(  self					:Any			,    _SCREAMING_SNAKE_CASE					:List[Any])					->			str:






								'''simple docstring'''






								UpperCamelCase_     :	List[str]					     =     """UNwant\u00E9d,running"""
								UpperCamelCase_     :	Union[str, Any]					     =     """unwanted, running"""
								return input_text, output_text




		def 	_snake_case	(  self					:Any)					->			Any:






								'''simple docstring'''






								UpperCamelCase_     :	str					     =     self.tokenizer_class(self.vocab_file)

								UpperCamelCase_     :	str					     =     tokenizer.tokenize("""UNwant\u00E9d,running""")
								self.assertListEqual(_SCREAMING_SNAKE_CASE			,    ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""])
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE)			,    [9, 6, 7, 1_2, 1_0, 1_1])




		def 	_snake_case	(  self					:Any)					->			Optional[Any]:






								'''simple docstring'''






								if not self.test_rust_tokenizer:
														return

								UpperCamelCase_     :	Optional[int]					     =     self.get_tokenizer()
								UpperCamelCase_     :	Union[str, Any]					     =     self.get_rust_tokenizer()

								UpperCamelCase_     :	List[Any]					     =     """UNwant\u00E9d,running"""

								UpperCamelCase_     :	Union[str, Any]					     =     tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
								UpperCamelCase_     :	List[Any]					     =     rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
								self.assertListEqual(_SCREAMING_SNAKE_CASE			,    _SCREAMING_SNAKE_CASE)

								UpperCamelCase_     :	List[Any]					     =     tokenizer.encode(_SCREAMING_SNAKE_CASE			,    add_special_tokens=_SCREAMING_SNAKE_CASE)
								UpperCamelCase_     :	Tuple					     =     rust_tokenizer.encode(_SCREAMING_SNAKE_CASE			,    add_special_tokens=_SCREAMING_SNAKE_CASE)
								self.assertListEqual(_SCREAMING_SNAKE_CASE			,    _SCREAMING_SNAKE_CASE)

								UpperCamelCase_     :	Dict					     =     self.get_rust_tokenizer()
								UpperCamelCase_     :	Optional[int]					     =     tokenizer.encode(_SCREAMING_SNAKE_CASE)
								UpperCamelCase_     :	List[Any]					     =     rust_tokenizer.encode(_SCREAMING_SNAKE_CASE)
								self.assertListEqual(_SCREAMING_SNAKE_CASE			,    _SCREAMING_SNAKE_CASE)

								# With lower casing
								UpperCamelCase_     :	Union[str, Any]					     =     self.get_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)
								UpperCamelCase_     :	List[Any]					     =     self.get_rust_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

								UpperCamelCase_     :	List[str]					     =     """UNwant\u00E9d,running"""

								UpperCamelCase_     :	Union[str, Any]					     =     tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
								UpperCamelCase_     :	List[str]					     =     rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
								self.assertListEqual(_SCREAMING_SNAKE_CASE			,    _SCREAMING_SNAKE_CASE)

								UpperCamelCase_     :	Union[str, Any]					     =     tokenizer.encode(_SCREAMING_SNAKE_CASE			,    add_special_tokens=_SCREAMING_SNAKE_CASE)
								UpperCamelCase_     :	str					     =     rust_tokenizer.encode(_SCREAMING_SNAKE_CASE			,    add_special_tokens=_SCREAMING_SNAKE_CASE)
								self.assertListEqual(_SCREAMING_SNAKE_CASE			,    _SCREAMING_SNAKE_CASE)

								UpperCamelCase_     :	Dict					     =     self.get_rust_tokenizer()
								UpperCamelCase_     :	Tuple					     =     tokenizer.encode(_SCREAMING_SNAKE_CASE)
								UpperCamelCase_     :	Dict					     =     rust_tokenizer.encode(_SCREAMING_SNAKE_CASE)
								self.assertListEqual(_SCREAMING_SNAKE_CASE			,    _SCREAMING_SNAKE_CASE)




		def 	_snake_case	(  self					:Tuple)					->			Optional[Any]:






								'''simple docstring'''






								UpperCamelCase_     :	Dict					     =     BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""")			,    ["""ah""", """\u535A""", """\u63A8""", """zz"""])




		def 	_snake_case	(  self					:Union[str, Any])					->			List[Any]:






								'''simple docstring'''






								UpperCamelCase_     :	List[str]					     =     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)			,    ["""hello""", """!""", """how""", """are""", """you""", """?"""])
								self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")			,    ["""hello"""])




		def 	_snake_case	(  self					:Dict)					->			List[str]:






								'''simple docstring'''






								UpperCamelCase_     :	List[Any]					     =     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE			,    strip_accents=_SCREAMING_SNAKE_CASE)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)			,    ["""hällo""", """!""", """how""", """are""", """you""", """?"""])
								self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")			,    ["""h\u00E9llo"""])




		def 	_snake_case	(  self					:int)					->			Dict:






								'''simple docstring'''






								UpperCamelCase_     :	Tuple					     =     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE			,    strip_accents=_SCREAMING_SNAKE_CASE)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)			,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""])
								self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")			,    ["""hello"""])




		def 	_snake_case	(  self					:Optional[int])					->			Any:






								'''simple docstring'''






								UpperCamelCase_     :	Union[str, Any]					     =     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)			,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""])
								self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")			,    ["""hello"""])




		def 	_snake_case	(  self					:str)					->			List[Any]:






								'''simple docstring'''






								UpperCamelCase_     :	Dict					     =     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)			,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""])




		def 	_snake_case	(  self					:Tuple)					->			Optional[Any]:






								'''simple docstring'''






								UpperCamelCase_     :	int					     =     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE			,    strip_accents=_SCREAMING_SNAKE_CASE)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)			,    ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""])




		def 	_snake_case	(  self					:int)					->			Optional[int]:






								'''simple docstring'''






								UpperCamelCase_     :	List[Any]					     =     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE			,    strip_accents=_SCREAMING_SNAKE_CASE)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)			,    ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""])




		def 	_snake_case	(  self					:Tuple)					->			Union[str, Any]:






								'''simple docstring'''






								UpperCamelCase_     :	str					     =     BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE			,    never_split=["""[UNK]"""])

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""")			,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""])




		def 	_snake_case	(  self					:str)					->			Optional[int]:






								'''simple docstring'''






								UpperCamelCase_     :	List[str]					     =     ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

								UpperCamelCase_     :	Optional[Any]					     =     {}
								for i, token in enumerate(_SCREAMING_SNAKE_CASE):
														UpperCamelCase_     :	int					     =     i
								UpperCamelCase_     :	Optional[int]					     =     WordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE			,    unk_token="""[UNK]""")

								self.assertListEqual(tokenizer.tokenize("""""")			,    [])

								self.assertListEqual(tokenizer.tokenize("""unwanted running""")			,    ["""un""", """##want""", """##ed""", """runn""", """##ing"""])

								self.assertListEqual(tokenizer.tokenize("""unwantedX running""")			,    ["""[UNK]""", """runn""", """##ing"""])




		def 	_snake_case	(  self					:List[Any])					->			Any:






								'''simple docstring'''






								self.assertTrue(_is_whitespace(""" """))
								self.assertTrue(_is_whitespace("""\t"""))
								self.assertTrue(_is_whitespace("""\r"""))
								self.assertTrue(_is_whitespace("""\n"""))
								self.assertTrue(_is_whitespace("""\u00A0"""))

								self.assertFalse(_is_whitespace("""A"""))
								self.assertFalse(_is_whitespace("""-"""))




		def 	_snake_case	(  self					:Tuple)					->			Optional[Any]:






								'''simple docstring'''






								self.assertTrue(_is_control("""\u0005"""))

								self.assertFalse(_is_control("""A"""))
								self.assertFalse(_is_control(""" """))
								self.assertFalse(_is_control("""\t"""))
								self.assertFalse(_is_control("""\r"""))




		def 	_snake_case	(  self					:Tuple)					->			List[Any]:






								'''simple docstring'''






								self.assertTrue(_is_punctuation("""-"""))
								self.assertTrue(_is_punctuation("""$"""))
								self.assertTrue(_is_punctuation("""`"""))
								self.assertTrue(_is_punctuation("""."""))

								self.assertFalse(_is_punctuation("""A"""))
								self.assertFalse(_is_punctuation(""" """))




		def 	_snake_case	(  self					:Optional[int])					->			List[Any]:






								'''simple docstring'''






								UpperCamelCase_     :	Tuple					     =     self.get_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(_SCREAMING_SNAKE_CASE) for t in ["""Test""", """\xad""", """test"""]]			,    [["""[UNK]"""], [], ["""[UNK]"""]])

								if self.test_rust_tokenizer:
														UpperCamelCase_     :	Dict					     =     self.get_rust_tokenizer()
														self.assertListEqual(
														    [rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE) for t in ["""Test""", """\xad""", """test"""]]			,    [["""[UNK]"""], [], ["""[UNK]"""]])




		@slow
		def 	_snake_case	(  self					:int)					->			List[Any]:






								'''simple docstring'''






								UpperCamelCase_     :	Union[str, Any]					     =     self.tokenizer_class.from_pretrained("""bert-base-uncased""")

								UpperCamelCase_     :	int					     =     tokenizer.encode("""sequence builders"""			,    add_special_tokens=_SCREAMING_SNAKE_CASE)
								UpperCamelCase_     :	Optional[int]					     =     tokenizer.encode("""multi-sequence build"""			,    add_special_tokens=_SCREAMING_SNAKE_CASE)

								UpperCamelCase_     :	Optional[Any]					     =     tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE)
								UpperCamelCase_     :	Dict					     =     tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE			,    _SCREAMING_SNAKE_CASE)

								assert encoded_sentence == [1_0_1] + text + [1_0_2]
								assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




		def 	_snake_case	(  self					:Dict)					->			Tuple:






								'''simple docstring'''






								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
																				UpperCamelCase_     :	Optional[int]					     =     self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE			,    **_SCREAMING_SNAKE_CASE)

																				UpperCamelCase_     :	List[Any]					     =     F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																				UpperCamelCase_     :	Optional[int]					     =     tokenizer_r.encode_plus(
																				    _SCREAMING_SNAKE_CASE			,    return_attention_mask=_SCREAMING_SNAKE_CASE			,    return_token_type_ids=_SCREAMING_SNAKE_CASE			,    return_offsets_mapping=_SCREAMING_SNAKE_CASE			,    add_special_tokens=_SCREAMING_SNAKE_CASE			,    )

																				UpperCamelCase_     :	Optional[int]					     =     tokenizer_r.do_lower_case if hasattr(_SCREAMING_SNAKE_CASE			,    """do_lower_case""") else False
																				UpperCamelCase_     :	List[str]					     =     (
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), """A"""),
																				        ((1, 2), ""","""),
																				        ((3, 5), """na"""),
																				        ((5, 6), """##ï"""),
																				        ((6, 8), """##ve"""),
																				        ((9, 1_5), tokenizer_r.mask_token),
																				        ((1_6, 2_1), """Allen"""),
																				        ((2_1, 2_3), """##NL"""),
																				        ((2_3, 2_4), """##P"""),
																				        ((2_5, 3_3), """sentence"""),
																				        ((3_3, 3_4), """."""),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), """a"""),
																				        ((1, 2), ""","""),
																				        ((3, 8), """naive"""),
																				        ((9, 1_5), tokenizer_r.mask_token),
																				        ((1_6, 2_1), """allen"""),
																				        ((2_1, 2_3), """##nl"""),
																				        ((2_3, 2_4), """##p"""),
																				        ((2_5, 3_3), """sentence"""),
																				        ((3_3, 3_4), """."""),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]			,    tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]))
																				self.assertEqual([e[0] for e in expected_results]			,    tokens["""offset_mapping"""])





		@slow
		def 	_snake_case	(  self					:Any)					->			str:






								'''simple docstring'''






								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
																				UpperCamelCase_     :	List[str]					     =     self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE			,    **_SCREAMING_SNAKE_CASE)
																				UpperCamelCase_     :	Union[str, Any]					     =     self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE			,    **_SCREAMING_SNAKE_CASE)
																				UpperCamelCase_     :	Optional[int]					     =     [["""Hello world!""", """Nice to meet you!"""], ["""The cute cat.""", """The adorable dog."""]]

																				UpperCamelCase_     :	Union[str, Any]					     =     tokenizer_r.batch_encode_candidates(_SCREAMING_SNAKE_CASE			,    max_length=1_0			,    return_tensors="""np""")
																				UpperCamelCase_     :	List[str]					     =     tokenizer_p.batch_encode_candidates(_SCREAMING_SNAKE_CASE			,    max_length=1_0			,    return_tensors="""np""")

																				UpperCamelCase_     :	Union[str, Any]					     =     (2, 2, 1_0)

																				self.assertEqual(encoded_sentence_r["""input_ids"""].shape			,    _SCREAMING_SNAKE_CASE)
																				self.assertEqual(encoded_sentence_r["""attention_mask"""].shape			,    _SCREAMING_SNAKE_CASE)
																				self.assertEqual(encoded_sentence_r["""token_type_ids"""].shape			,    _SCREAMING_SNAKE_CASE)

																				self.assertEqual(encoded_sentence_p["""input_ids"""].shape			,    _SCREAMING_SNAKE_CASE)
																				self.assertEqual(encoded_sentence_p["""attention_mask"""].shape			,    _SCREAMING_SNAKE_CASE)
																				self.assertEqual(encoded_sentence_p["""token_type_ids"""].shape			,    _SCREAMING_SNAKE_CASE)




