





'''simple docstring'''






import inspect
import os
from argparse import ArgumentParser, Namespace
from importlib import import_module

import huggingface_hub
import numpy as np
from packaging import version

from .. import (
    FEATURE_EXTRACTOR_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoTokenizer,
    is_datasets_available,
    is_tf_available,
    is_torch_available,
)
from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
from . import BaseTransformersCLICommand


if is_tf_available():
	import tensorflow as tf

	tf.config.experimental.enable_tensor_float_aa_execution(False)

if is_torch_available():
	import torch

if is_datasets_available():
	from datasets import load_dataset


__UpperCamelCase										=  5E-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors







def 				_a	(					_lowerCamelCase   )   ->							Union[str, Any]:



			"""simple docstring"""


			return PTtoTFCommand(
			    args.model_name						,  args.local_dir						,  args.max_error						,  args.new_weights						,  args.no_pr						,  args.push						,  args.extra_commit_description						,  args.override_model_class						,  )




class    _A   (				__lowercase		):

		@staticmethod
		def   lowercase__    (      __magic_name__				:      ArgumentParser )				->						List[Any]:


					"""simple docstring"""

					__snake_case				:			int       			=    parser.add_parser(
					    """pt-to-tf"""     ,			help=(
					        """CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."""
					        """ Can also be used to validate existing weights without opening PRs, with --no-pr."""
					    )     ,			)
					train_parser.add_argument(
					    """--model-name"""     ,			type=__magic_name__     ,			required=__magic_name__     ,			help="""The model name, including owner/organization, as seen on the hub."""     ,			)
					train_parser.add_argument(
					    """--local-dir"""     ,			type=__magic_name__     ,			default=""""""     ,			help="""Optional local directory of the model repository. Defaults to /tmp/{model_name}"""     ,			)
					train_parser.add_argument(
					    """--max-error"""     ,			type=__magic_name__     ,			default=__magic_name__     ,			help=(
					        f'''Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk.'''
					    )     ,			)
					train_parser.add_argument(
					    """--new-weights"""     ,			action="""store_true"""     ,			help="""Optional flag to create new TensorFlow weights, even if they already exist."""     ,			)
					train_parser.add_argument(
					    """--no-pr"""     ,			action="""store_true"""     ,			help="""Optional flag to NOT open a PR with converted weights.""" )
					train_parser.add_argument(
					    """--push"""     ,			action="""store_true"""     ,			help="""Optional flag to push the weights directly to `main` (requires permissions)"""     ,			)
					train_parser.add_argument(
					    """--extra-commit-description"""     ,			type=__magic_name__     ,			default=""""""     ,			help="""Optional additional commit description to use when opening a PR (e.g. to tag the owner)."""     ,			)
					train_parser.add_argument(
					    """--override-model-class"""     ,			type=__magic_name__     ,			default=__magic_name__     ,			help="""If you think you know better than the auto-detector, you can specify the model class here. """
					    """Can be either an AutoModel class or a specific model class like BertForSequenceClassification."""     ,			)
					train_parser.set_defaults(func=__magic_name__ )

		@staticmethod
		def   lowercase__    (      __magic_name__				:      List[str]     ,			__magic_name__				:      str )				->						Any:


					"""simple docstring"""

					__snake_case				:			Dict       			=    set(pt_outputs.keys() )
					__snake_case				:			Dict       			=    set(tf_outputs.keys() )
					if pt_out_attrs != tf_out_attrs:
								raise ValueError(
								    f'''The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:'''
								    f''' {tf_out_attrs})''' )

					# 2. For each output attribute, computes the difference
					def _find_pt_tf_differences(__magic_name__				:      List[Any]     ,			__magic_name__				:      List[str]     ,			__magic_name__				:      List[Any]     ,			__magic_name__				:      Any="" ):
								# If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
								# recursivelly, keeping the name of the attribute.
								if isinstance(__magic_name__     ,			torch.Tensor ):
											__snake_case				:			Any       			=    np.max(np.abs(pt_out.numpy() - tf_out.numpy() ) )
											__snake_case				:			Optional[Any]       			=    tensor_difference
								else:
											__snake_case				:			Optional[Any]       			=    attr_name
											for i, pt_item in enumerate(__magic_name__ ):
														# If it is a named attribute, we keep the name. Otherwise, just its index.
														if isinstance(__magic_name__     ,			__magic_name__ ):
																	__snake_case				:			str       			=    root_name + pt_item
																	__snake_case				:			List[Any]       			=    tf_out[pt_item]
																	__snake_case				:			Optional[int]       			=    pt_out[pt_item]
														else:
																	__snake_case				:			Optional[Any]       			=    root_name + f'''[{i}]'''
																	__snake_case				:			int       			=    tf_out[i]
														__snake_case				:			int       			=    _find_pt_tf_differences(__magic_name__     ,			__magic_name__     ,			__magic_name__     ,			__magic_name__ )

								return differences

					return _find_pt_tf_differences(__magic_name__     ,			__magic_name__     ,			{} )

		def __init__(      self				:      str     ,			__magic_name__				:      str     ,			__magic_name__				:      str     ,			__magic_name__				:      float     ,			__magic_name__				:      bool     ,			__magic_name__				:      bool     ,			__magic_name__				:      bool     ,			__magic_name__				:      str     ,			__magic_name__				:      str     ,			*__magic_name__				:      int     ,			)				->						List[Any]:


					"""simple docstring"""

					__snake_case				:			Any       			=    logging.get_logger("""transformers-cli/pt_to_tf""" )
					__snake_case				:			int       			=    model_name
					__snake_case				:			Optional[Any]       			=    local_dir if local_dir else os.path.join("""/tmp"""     ,			__magic_name__ )
					__snake_case				:			Dict       			=    max_error
					__snake_case				:			List[str]       			=    new_weights
					__snake_case				:			int       			=    no_pr
					__snake_case				:			int       			=    push
					__snake_case				:			Any       			=    extra_commit_description
					__snake_case				:			Dict       			=    override_model_class

		def   lowercase__    (      self				:      Dict     ,			__magic_name__				:      Dict     ,			__magic_name__				:      List[str]     ,			__magic_name__				:      List[Any] )				->						List[Any]:


					"""simple docstring"""

					def _get_audio_input():
								__snake_case				:			Tuple       			=    load_dataset("""hf-internal-testing/librispeech_asr_dummy"""     ,			"""clean"""     ,			split="""validation""" )
								__snake_case				:			Dict       			=    ds.sort("""id""" ).select(range(2 ) )[:2]["""audio"""]
								__snake_case				:			List[str]       			=    [x["""array"""] for x in speech_samples]
								return raw_samples

					__snake_case				:			List[Any]       			=    type(pt_model.config )
					if model_config_class in PROCESSOR_MAPPING:
								__snake_case				:			Optional[Any]       			=    AutoProcessor.from_pretrained(self._local_dir )
								if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
											__snake_case				:			Optional[Any]       			=    processor.tokenizer.eos_token
					elif model_config_class in IMAGE_PROCESSOR_MAPPING:
								__snake_case				:			Union[str, Any]       			=    AutoImageProcessor.from_pretrained(self._local_dir )
					elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
								__snake_case				:			List[Any]       			=    AutoFeatureExtractor.from_pretrained(self._local_dir )
					elif model_config_class in TOKENIZER_MAPPING:
								__snake_case				:			Tuple       			=    AutoTokenizer.from_pretrained(self._local_dir )
								if processor.pad_token is None:
											__snake_case				:			Optional[int]       			=    processor.eos_token
					else:
								raise ValueError(f'''Unknown data processing type (model config type: {model_config_class})''' )

					__snake_case				:			int       			=    set(inspect.signature(pt_model.forward ).parameters.keys() )
					__snake_case				:			Dict       			=    {}
					if "input_ids" in model_forward_signature:
								processor_inputs.update(
								    {
								        """text""": ["""Hi there!""", """I am a batch with more than one row and different input lengths."""],
								        """padding""": True,
								        """truncation""": True,
								    } )
					if "pixel_values" in model_forward_signature:
								__snake_case				:			Tuple       			=    load_dataset("""cifar10"""     ,			"""plain_text"""     ,			split="""test""" )[:2]["""img"""]
								processor_inputs.update({"""images""": sample_images} )
					if "input_features" in model_forward_signature:
								__snake_case				:			int       			=    inspect.signature(processor.feature_extractor ).parameters
								# Pad to the largest input length by default but take feature extractor default
								# padding value if it exists e.g. "max_length" and is not False or None
								if "padding" in feature_extractor_signature:
											__snake_case				:			Optional[int]       			=    feature_extractor_signature["""padding"""].default
											if default_strategy is not False and default_strategy is not None:
														__snake_case				:			Union[str, Any]       			=    default_strategy
											else:
														__snake_case				:			List[Any]       			=    True
								else:
											__snake_case				:			Any       			=    True
								processor_inputs.update({"""audio""": _get_audio_input(), """padding""": padding_strategy} )
					if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
								processor_inputs.update({"""audio""": _get_audio_input(), """padding""": True} )
					__snake_case				:			Optional[Any]       			=    processor(**__magic_name__     ,			return_tensors="""pt""" )
					__snake_case				:			Any       			=    processor(**__magic_name__     ,			return_tensors="""tf""" )

					# Extra input requirements, in addition to the input modality
					if (
					    config.is_encoder_decoder
					    or (hasattr(__magic_name__     ,			"""encoder""" ) and hasattr(__magic_name__     ,			"""decoder""" ))
					    or "decoder_input_ids" in tf_dummy_inputs
					):
								__snake_case				:			Union[str, Any]       			=    np.asarray([[1], [1]]     ,			dtype=__magic_name__ ) * (pt_model.config.decoder_start_token_id or 0)
								pt_input.update({"""decoder_input_ids""": torch.tensor(__magic_name__ )} )
								tf_input.update({"""decoder_input_ids""": tf.convert_to_tensor(__magic_name__ )} )

					return pt_input, tf_input

		def   lowercase__    (      self				:      Dict )				->						Tuple:


					"""simple docstring"""

					if version.parse(huggingface_hub.__version__ ) < version.parse("""0.9.0""" ):
								raise ImportError(
								    """The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"""
								    """ installation.""" )
					else:
								from huggingface_hub import Repository, create_commit
								from huggingface_hub._commit_api import CommitOperationAdd

					# Fetch remote data
					__snake_case				:			Dict       			=    Repository(local_dir=self._local_dir     ,			clone_from=self._model_name )

					# Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
					__snake_case				:			Optional[int]       			=    AutoConfig.from_pretrained(self._local_dir )
					__snake_case				:			Union[str, Any]       			=    config.architectures
					if self._override_model_class is not None:
								if self._override_model_class.startswith("""TF""" ):
											__snake_case				:			Any       			=    [self._override_model_class[2:]]
								else:
											__snake_case				:			List[str]       			=    [self._override_model_class]
								try:
											__snake_case				:			Union[str, Any]       			=    getattr(import_module("""transformers""" )     ,			architectures[0] )
								except AttributeError:
											raise ValueError(f'''Model class {self._override_model_class} not found in transformers.''' )
								try:
											__snake_case				:			List[Any]       			=    getattr(import_module("""transformers""" )     ,			"""TF""" + architectures[0] )
								except AttributeError:
											raise ValueError(f'''TF model class TF{self._override_model_class} not found in transformers.''' )
					elif architectures is None:  # No architecture defined -- use auto classes
								__snake_case				:			int       			=    getattr(import_module("""transformers""" )     ,			"""AutoModel""" )
								__snake_case				:			List[Any]       			=    getattr(import_module("""transformers""" )     ,			"""TFAutoModel""" )
								self._logger.warning("""No detected architecture, using AutoModel/TFAutoModel""" )
					else:  # Architecture defined -- use it
								if len(__magic_name__ ) > 1:
											raise ValueError(f'''More than one architecture was found, aborting. (architectures = {architectures})''' )
								self._logger.warning(f'''Detected architecture: {architectures[0]}''' )
								__snake_case				:			Dict       			=    getattr(import_module("""transformers""" )     ,			architectures[0] )
								try:
											__snake_case				:			Tuple       			=    getattr(import_module("""transformers""" )     ,			"""TF""" + architectures[0] )
								except AttributeError:
											raise AttributeError(f'''The TensorFlow equivalent of {architectures[0]} doesn\'t exist in transformers.''' )

        # Check the TF dummy inputs to see what keys we need in the forward pass
					__snake_case				:			List[str]       			=    tf_class.from_config(__magic_name__ )
					__snake_case				:			str       			=    tf_from_pt_model.dummy_inputs

					del tf_from_pt_model  # Try to keep only one model in memory at a time

					# Load the model and get some basic inputs
					__snake_case				:			int       			=    pt_class.from_pretrained(self._local_dir )
					pt_model.eval()

					__snake_case			,    __snake_case				:			int       			=    self.get_inputs(__magic_name__     ,			__magic_name__     ,			__magic_name__ )

					with torch.no_grad():
								__snake_case				:			Union[str, Any]       			=    pt_model(**__magic_name__     ,			output_hidden_states=__magic_name__ )
					del pt_model  # will no longer be used, and may have a large memory footprint

					__snake_case				:			Optional[Any]       			=    tf_class.from_pretrained(self._local_dir     ,			from_pt=__magic_name__ )
					__snake_case				:			Tuple       			=    tf_from_pt_model(**__magic_name__     ,			output_hidden_states=__magic_name__     ,			training=__magic_name__ )

					# Confirms that cross loading PT weights into TF worked.
					__snake_case				:			Union[str, Any]       			=    self.find_pt_tf_differences(__magic_name__     ,			__magic_name__ )
					__snake_case				:			str       			=    {k: v for k, v in crossload_differences.items() if """hidden""" not in k}
					__snake_case				:			Dict       			=    {k: v for k, v in crossload_differences.items() if """hidden""" in k}
					if len(__magic_name__ ) == 0 and architectures is not None:
								raise ValueError(
								    f'''Something went wrong -- the config file has architectures ({architectures}), but no model head'''
								    """ output was found. All outputs start with 'hidden'""" )
					__snake_case				:			Dict       			=    max(output_differences.values() ) if output_differences else 0.0
					__snake_case				:			Tuple       			=    max(hidden_differences.values() )
					if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
								raise ValueError(
								    """The cross-loaded TensorFlow model has different outputs, something went wrong!\n"""
								    + f'''\nList of maximum output differences above the threshold ({self._max_error}):\n'''
								    + """\n""".join([f'''{k}: {v:.3e}''' for k, v in output_differences.items() if v > self._max_error] )
								    + f'''\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'''
								    + """\n""".join([f'''{k}: {v:.3e}''' for k, v in hidden_differences.items() if v > self._max_error] ) )

					# Save the weights in a TF format (if needed) and confirms that the results are still good
					__snake_case				:			Dict       			=    os.path.join(self._local_dir     ,			__magic_name__ )
					__snake_case				:			Optional[Any]       			=    os.path.join(self._local_dir     ,			__magic_name__ )
					if (not os.path.exists(__magic_name__ ) and not os.path.exists(__magic_name__ )) or self._new_weights:
								tf_from_pt_model.save_pretrained(self._local_dir )
					del tf_from_pt_model  # will no longer be used, and may have a large memory footprint

					__snake_case				:			Tuple       			=    tf_class.from_pretrained(self._local_dir )
					__snake_case				:			List[str]       			=    tf_model(**__magic_name__     ,			output_hidden_states=__magic_name__ )

					__snake_case				:			int       			=    self.find_pt_tf_differences(__magic_name__     ,			__magic_name__ )
					__snake_case				:			Any       			=    {k: v for k, v in conversion_differences.items() if """hidden""" not in k}
					__snake_case				:			int       			=    {k: v for k, v in conversion_differences.items() if """hidden""" in k}
					if len(__magic_name__ ) == 0 and architectures is not None:
								raise ValueError(
								    f'''Something went wrong -- the config file has architectures ({architectures}), but no model head'''
								    """ output was found. All outputs start with 'hidden'""" )
					__snake_case				:			Optional[Any]       			=    max(output_differences.values() ) if output_differences else 0.0
					__snake_case				:			int       			=    max(hidden_differences.values() )
					if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
								raise ValueError(
								    """The converted TensorFlow model has different outputs, something went wrong!\n"""
								    + f'''\nList of maximum output differences above the threshold ({self._max_error}):\n'''
								    + """\n""".join([f'''{k}: {v:.3e}''' for k, v in output_differences.items() if v > self._max_error] )
								    + f'''\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'''
								    + """\n""".join([f'''{k}: {v:.3e}''' for k, v in hidden_differences.items() if v > self._max_error] ) )

					__snake_case				:			Tuple       			=    """Update TF weights""" if self._new_weights else """Add TF weights"""
					if self._push:
								repo.git_add(auto_lfs_track=__magic_name__ )
								repo.git_commit(__magic_name__ )
								repo.git_push(blocking=__magic_name__ )  # this prints a progress bar with the upload
								self._logger.warning(f'''TF weights pushed into {self._model_name}''' )
					elif not self._no_pr:
								self._logger.warning("""Uploading the weights into a new PR...""" )
								__snake_case				:			Union[str, Any]       			=    (
								    """Model converted by the [`transformers`' `pt_to_tf`"""
								    """ CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). """
								    """All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n"""
								    f'''Maximum crossload output difference={max_crossload_output_diff:.3e}; '''
								    f'''Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n'''
								    f'''Maximum conversion output difference={max_conversion_output_diff:.3e}; '''
								    f'''Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n'''
								)
								if self._max_error > MAX_ERROR:
											commit_descrition += (
											    f'''\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!'''
											)
								if self._extra_commit_description:
											commit_descrition += "\n\n" + self._extra_commit_description

								# sharded model -> adds all related files (index and .h5 shards)
								if os.path.exists(__magic_name__ ):
											__snake_case				:			Optional[int]       			=    [
											    CommitOperationAdd(path_in_repo=__magic_name__     ,			path_or_fileobj=__magic_name__ )
											]
											for shard_path in tf.io.gfile.glob(self._local_dir + """/tf_model-*.h5""" ):
														operations += [
														    CommitOperationAdd(path_in_repo=os.path.basename(__magic_name__ )     ,			path_or_fileobj=__magic_name__ )
														]
								else:
											__snake_case				:			Dict       			=    [CommitOperationAdd(path_in_repo=__magic_name__     ,			path_or_fileobj=__magic_name__ )]

								__snake_case				:			Any       			=    create_commit(
								    repo_id=self._model_name     ,			operations=__magic_name__     ,			commit_message=__magic_name__     ,			commit_description=__magic_name__     ,			repo_type="""model"""     ,			create_pr=__magic_name__     ,			).pr_url
								self._logger.warning(f'''PR open in {hub_pr_url}''' )









'''simple docstring'''






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class    _A   (				__lowercase	,				unittest.TestCase		):
		lowercase__:      str							    =      ElectraTokenizer
		lowercase__:      Dict							    =      ElectraTokenizerFast
		lowercase__:      Union[str, Any]							    =      True
		lowercase__:      Tuple							    =      True
		lowercase__:      int							    =      filter_non_english

		def   lowercase__    (      self				:      Tuple )				->						Tuple:


					"""simple docstring"""

					super().setUp()

					__snake_case				:			Union[str, Any]       			=    [
					    """[UNK]""",
					    """[CLS]""",
					    """[SEP]""",
					    """[PAD]""",
					    """[MASK]""",
					    """want""",
					    """##want""",
					    """##ed""",
					    """wa""",
					    """un""",
					    """runn""",
					    """##ing""",
					    """,""",
					    """low""",
					    """lowest""",
					]
					__snake_case				:			List[Any]       			=    os.path.join(self.tmpdirname     ,			VOCAB_FILES_NAMES["""vocab_file"""] )
					with open(self.vocab_file     ,			"""w"""     ,			encoding="""utf-8""" ) as vocab_writer:
								vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens] ) )

		def   lowercase__    (      self				:      List[Any]     ,			__magic_name__				:      str )				->						str:


					"""simple docstring"""

					__snake_case				:			List[str]       			=    """UNwant\u00E9d,running"""
					__snake_case				:			Any       			=    """unwanted, running"""
					return input_text, output_text

		def   lowercase__    (      self				:      Union[str, Any] )				->						List[str]:


					"""simple docstring"""

					__snake_case				:			Tuple       			=    self.tokenizer_class(self.vocab_file )

					__snake_case				:			Dict       			=    tokenizer.tokenize("""UNwant\u00E9d,running""" )
					self.assertListEqual(__magic_name__     ,			["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""] )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__ )     ,			[9, 6, 7, 12, 10, 11] )

		def   lowercase__    (      self				:      Dict )				->						int:


					"""simple docstring"""

					if not self.test_rust_tokenizer:
								return

					__snake_case				:			str       			=    self.get_tokenizer()
					__snake_case				:			Union[str, Any]       			=    self.get_rust_tokenizer()

					__snake_case				:			Tuple       			=    """UNwant\u00E9d,running"""

					__snake_case				:			List[str]       			=    tokenizer.tokenize(__magic_name__ )
					__snake_case				:			Dict       			=    rust_tokenizer.tokenize(__magic_name__ )
					self.assertListEqual(__magic_name__     ,			__magic_name__ )

					__snake_case				:			int       			=    tokenizer.encode(__magic_name__     ,			add_special_tokens=__magic_name__ )
					__snake_case				:			str       			=    rust_tokenizer.encode(__magic_name__     ,			add_special_tokens=__magic_name__ )
					self.assertListEqual(__magic_name__     ,			__magic_name__ )

					__snake_case				:			Dict       			=    self.get_rust_tokenizer()
					__snake_case				:			Optional[int]       			=    tokenizer.encode(__magic_name__ )
					__snake_case				:			int       			=    rust_tokenizer.encode(__magic_name__ )
					self.assertListEqual(__magic_name__     ,			__magic_name__ )

					# With lower casing
					__snake_case				:			str       			=    self.get_tokenizer(do_lower_case=__magic_name__ )
					__snake_case				:			str       			=    self.get_rust_tokenizer(do_lower_case=__magic_name__ )

					__snake_case				:			int       			=    """UNwant\u00E9d,running"""

					__snake_case				:			int       			=    tokenizer.tokenize(__magic_name__ )
					__snake_case				:			List[str]       			=    rust_tokenizer.tokenize(__magic_name__ )
					self.assertListEqual(__magic_name__     ,			__magic_name__ )

					__snake_case				:			Any       			=    tokenizer.encode(__magic_name__     ,			add_special_tokens=__magic_name__ )
					__snake_case				:			Tuple       			=    rust_tokenizer.encode(__magic_name__     ,			add_special_tokens=__magic_name__ )
					self.assertListEqual(__magic_name__     ,			__magic_name__ )

					__snake_case				:			List[Any]       			=    self.get_rust_tokenizer()
					__snake_case				:			Tuple       			=    tokenizer.encode(__magic_name__ )
					__snake_case				:			Optional[int]       			=    rust_tokenizer.encode(__magic_name__ )
					self.assertListEqual(__magic_name__     ,			__magic_name__ )

		def   lowercase__    (      self				:      Union[str, Any] )				->						str:


					"""simple docstring"""

					__snake_case				:			Union[str, Any]       			=    BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""" )     ,			["""ah""", """\u535A""", """\u63A8""", """zz"""] )

		def   lowercase__    (      self				:      Tuple )				->						Tuple:


					"""simple docstring"""

					__snake_case				:			List[Any]       			=    BasicTokenizer(do_lower_case=__magic_name__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )     ,			["""hello""", """!""", """how""", """are""", """you""", """?"""] )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )     ,			["""hello"""] )

		def   lowercase__    (      self				:      Optional[Any] )				->						Tuple:


					"""simple docstring"""

					__snake_case				:			Tuple       			=    BasicTokenizer(do_lower_case=__magic_name__     ,			strip_accents=__magic_name__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )     ,			["""hällo""", """!""", """how""", """are""", """you""", """?"""] )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )     ,			["""h\u00E9llo"""] )

		def   lowercase__    (      self				:      Optional[int] )				->						Any:


					"""simple docstring"""

					__snake_case				:			List[Any]       			=    BasicTokenizer(do_lower_case=__magic_name__     ,			strip_accents=__magic_name__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )     ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )     ,			["""hello"""] )

		def   lowercase__    (      self				:      Any )				->						Optional[Any]:


					"""simple docstring"""

					__snake_case				:			Any       			=    BasicTokenizer(do_lower_case=__magic_name__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )     ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )     ,			["""hello"""] )

		def   lowercase__    (      self				:      Dict )				->						int:


					"""simple docstring"""

					__snake_case				:			Any       			=    BasicTokenizer(do_lower_case=__magic_name__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )     ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )

		def   lowercase__    (      self				:      Union[str, Any] )				->						Tuple:


					"""simple docstring"""

					__snake_case				:			str       			=    BasicTokenizer(do_lower_case=__magic_name__     ,			strip_accents=__magic_name__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )     ,			["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )

		def   lowercase__    (      self				:      str )				->						int:


					"""simple docstring"""

					__snake_case				:			str       			=    BasicTokenizer(do_lower_case=__magic_name__     ,			strip_accents=__magic_name__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )     ,			["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )

		def   lowercase__    (      self				:      Optional[Any] )				->						Optional[Any]:


					"""simple docstring"""

					__snake_case				:			Tuple       			=    BasicTokenizer(do_lower_case=__magic_name__     ,			never_split=["""[UNK]"""] )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""" )     ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""] )

		def   lowercase__    (      self				:      List[str] )				->						Tuple:


					"""simple docstring"""

					__snake_case				:			str       			=    ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

					__snake_case				:			Union[str, Any]       			=    {}
					for i, token in enumerate(__magic_name__ ):
								__snake_case				:			List[str]       			=    i
					__snake_case				:			Union[str, Any]       			=    WordpieceTokenizer(vocab=__magic_name__     ,			unk_token="""[UNK]""" )

					self.assertListEqual(tokenizer.tokenize("""""" )     ,			[] )

					self.assertListEqual(tokenizer.tokenize("""unwanted running""" )     ,			["""un""", """##want""", """##ed""", """runn""", """##ing"""] )

					self.assertListEqual(tokenizer.tokenize("""unwantedX running""" )     ,			["""[UNK]""", """runn""", """##ing"""] )

		def   lowercase__    (      self				:      str )				->						Dict:


					"""simple docstring"""

					self.assertTrue(_is_whitespace(""" """ ) )
					self.assertTrue(_is_whitespace("""\t""" ) )
					self.assertTrue(_is_whitespace("""\r""" ) )
					self.assertTrue(_is_whitespace("""\n""" ) )
					self.assertTrue(_is_whitespace("""\u00A0""" ) )

					self.assertFalse(_is_whitespace("""A""" ) )
					self.assertFalse(_is_whitespace("""-""" ) )

		def   lowercase__    (      self				:      Optional[int] )				->						Optional[int]:


					"""simple docstring"""

					self.assertTrue(_is_control("""\u0005""" ) )

					self.assertFalse(_is_control("""A""" ) )
					self.assertFalse(_is_control(""" """ ) )
					self.assertFalse(_is_control("""\t""" ) )
					self.assertFalse(_is_control("""\r""" ) )

		def   lowercase__    (      self				:      Tuple )				->						Dict:


					"""simple docstring"""

					self.assertTrue(_is_punctuation("""-""" ) )
					self.assertTrue(_is_punctuation("""$""" ) )
					self.assertTrue(_is_punctuation("""`""" ) )
					self.assertTrue(_is_punctuation(""".""" ) )

					self.assertFalse(_is_punctuation("""A""" ) )
					self.assertFalse(_is_punctuation(""" """ ) )

		def   lowercase__    (      self				:      Union[str, Any] )				->						Optional[Any]:


					"""simple docstring"""

					__snake_case				:			Union[str, Any]       			=    self.get_tokenizer()
					__snake_case				:			Union[str, Any]       			=    self.get_rust_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(__magic_name__ ) for t in ["""Test""", """\xad""", """test"""]]     ,			[["""[UNK]"""], [], ["""[UNK]"""]] )

					self.assertListEqual(
					    [rust_tokenizer.tokenize(__magic_name__ ) for t in ["""Test""", """\xad""", """test"""]]     ,			[["""[UNK]"""], [], ["""[UNK]"""]] )

		@slow
		def   lowercase__    (      self				:      int )				->						Dict:


					"""simple docstring"""

					__snake_case				:			Tuple       			=    self.tokenizer_class.from_pretrained("""google/electra-base-discriminator""" )

					__snake_case				:			Optional[int]       			=    tokenizer.encode("""sequence builders"""     ,			add_special_tokens=__magic_name__ )
					__snake_case				:			Optional[Any]       			=    tokenizer.encode("""multi-sequence build"""     ,			add_special_tokens=__magic_name__ )

					__snake_case				:			List[str]       			=    tokenizer.build_inputs_with_special_tokens(__magic_name__ )
					__snake_case				:			Any       			=    tokenizer.build_inputs_with_special_tokens(__magic_name__     ,			__magic_name__ )

					assert encoded_sentence == [1_01] + text + [1_02]
					assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]

		def   lowercase__    (      self				:      List[str] )				->						Any:


					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
											__snake_case				:			Tuple       			=    self.rust_tokenizer_class.from_pretrained(__magic_name__     ,			**__magic_name__ )

											__snake_case				:			int       			=    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
											__snake_case				:			Optional[Any]       			=    tokenizer_r.encode_plus(
											    __magic_name__     ,			return_attention_mask=__magic_name__     ,			return_token_type_ids=__magic_name__     ,			return_offsets_mapping=__magic_name__     ,			add_special_tokens=__magic_name__     ,			)

											__snake_case				:			Optional[int]       			=    tokenizer_r.do_lower_case if hasattr(__magic_name__     ,			"""do_lower_case""" ) else False
											__snake_case				:			List[str]       			=    (
											    [
											        ((0, 0), tokenizer_r.cls_token),
											        ((0, 1), """A"""),
											        ((1, 2), ""","""),
											        ((3, 5), """na"""),
											        ((5, 6), """##ï"""),
											        ((6, 8), """##ve"""),
											        ((9, 15), tokenizer_r.mask_token),
											        ((16, 21), """Allen"""),
											        ((21, 23), """##NL"""),
											        ((23, 24), """##P"""),
											        ((25, 33), """sentence"""),
											        ((33, 34), """."""),
											        ((0, 0), tokenizer_r.sep_token),
											    ]
											    if not do_lower_case
											    else [
											        ((0, 0), tokenizer_r.cls_token),
											        ((0, 1), """a"""),
											        ((1, 2), ""","""),
											        ((3, 8), """naive"""),
											        ((9, 15), tokenizer_r.mask_token),
											        ((16, 21), """allen"""),
											        ((21, 23), """##nl"""),
											        ((23, 24), """##p"""),
											        ((25, 33), """sentence"""),
											        ((33, 34), """."""),
											        ((0, 0), tokenizer_r.sep_token),
											    ]
											)

											self.assertEqual(
											    [e[1] for e in expected_results]     ,			tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""] ) )
											self.assertEqual([e[0] for e in expected_results]     ,			tokens["""offset_mapping"""] )

		def   lowercase__    (      self				:      Optional[int] )				->						int:


					"""simple docstring"""

					__snake_case				:			Any       			=    ["""的""", """人""", """有"""]
					__snake_case				:			Union[str, Any]       			=    """""".join(__magic_name__ )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
											__snake_case				:			Tuple       			=    True
											__snake_case				:			List[str]       			=    self.tokenizer_class.from_pretrained(__magic_name__     ,			**__magic_name__ )
											__snake_case				:			Tuple       			=    self.rust_tokenizer_class.from_pretrained(__magic_name__     ,			**__magic_name__ )

											__snake_case				:			Tuple       			=    tokenizer_p.encode(__magic_name__     ,			add_special_tokens=__magic_name__ )
											__snake_case				:			Optional[int]       			=    tokenizer_r.encode(__magic_name__     ,			add_special_tokens=__magic_name__ )

											__snake_case				:			Union[str, Any]       			=    tokenizer_r.convert_ids_to_tokens(__magic_name__ )
											__snake_case				:			Tuple       			=    tokenizer_p.convert_ids_to_tokens(__magic_name__ )

											# it is expected that each Chinese character is not preceded by "##"
											self.assertListEqual(__magic_name__     ,			__magic_name__ )
											self.assertListEqual(__magic_name__     ,			__magic_name__ )

											__snake_case				:			Dict       			=    False
											__snake_case				:			str       			=    self.rust_tokenizer_class.from_pretrained(__magic_name__     ,			**__magic_name__ )
											__snake_case				:			Tuple       			=    self.tokenizer_class.from_pretrained(__magic_name__     ,			**__magic_name__ )

											__snake_case				:			List[Any]       			=    tokenizer_r.encode(__magic_name__     ,			add_special_tokens=__magic_name__ )
											__snake_case				:			str       			=    tokenizer_p.encode(__magic_name__     ,			add_special_tokens=__magic_name__ )

											__snake_case				:			Optional[Any]       			=    tokenizer_r.convert_ids_to_tokens(__magic_name__ )
											__snake_case				:			List[Any]       			=    tokenizer_p.convert_ids_to_tokens(__magic_name__ )

											# it is expected that only the first Chinese character is not preceded by "##".
											__snake_case				:			Dict       			=    [
											    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__magic_name__ )
											]
											self.assertListEqual(__magic_name__     ,			__magic_name__ )
											self.assertListEqual(__magic_name__     ,			__magic_name__ )


