import inspect
import os
from argparse import ArgumentParser, Namespace
from importlib import import_module

import huggingface_hub
import numpy as np
from packaging import version

from .. import (
    FEATURE_EXTRACTOR_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoTokenizer,
    is_datasets_available,
    is_tf_available,
    is_torch_available,
)
from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
from . import BaseTransformersCLICommand


if is_tf_available():
	import tensorflow as tf

	tf.config.experimental.enable_tensor_float_aa_execution(False)

if is_torch_available():
	import torch

if is_datasets_available():
	from datasets import load_dataset


_snake_case			=						5e-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors




def   _UpperCamelCase   (			snake_case__							)						->   List[Any]:
			return PTtoTFCommand(
			    args.model_name,    args.local_dir,    args.max_error,    args.new_weights,    args.no_pr,    args.push,    args.extra_commit_description,    args.override_model_class,    )







class 					_snake_case						(						UpperCAmelCase_				):
					@staticmethod
					def    _lowerCamelCase (  __lowerCamelCase:				Tuple     )    ->				Dict:
								__UpperCAmelCase						:			Dict         =   parser.add_parser(
								    "pt-to-tf"	,  help=(
								        "CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."
								        " Can also be used to validate existing weights without opening PRs, with --no-pr."
								    )	,  )
								train_parser.add_argument(
								    "--model-name"	,  type=__lowerCamelCase	,  required=__lowerCamelCase	,  help="The model name, including owner/organization, as seen on the hub."	,  )
								train_parser.add_argument(
								    "--local-dir"	,  type=__lowerCamelCase	,  default=""	,  help="Optional local directory of the model repository. Defaults to /tmp/{model_name}"	,  )
								train_parser.add_argument(
								    "--max-error"	,  type=__lowerCamelCase	,  default=__lowerCamelCase	,  help=(
								        f'''Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk.'''
								    )	,  )
								train_parser.add_argument(
								    "--new-weights"	,  action="store_true"	,  help="Optional flag to create new TensorFlow weights, even if they already exist."	,  )
								train_parser.add_argument(
								    "--no-pr"	,  action="store_true"	,  help="Optional flag to NOT open a PR with converted weights."     )
								train_parser.add_argument(
								    "--push"	,  action="store_true"	,  help="Optional flag to push the weights directly to `main` (requires permissions)"	,  )
								train_parser.add_argument(
								    "--extra-commit-description"	,  type=__lowerCamelCase	,  default=""	,  help="Optional additional commit description to use when opening a PR (e.g. to tag the owner)."	,  )
								train_parser.add_argument(
								    "--override-model-class"	,  type=__lowerCamelCase	,  default=__lowerCamelCase	,  help="If you think you know better than the auto-detector, you can specify the model class here. "
								    "Can be either an AutoModel class or a specific model class like BertForSequenceClassification."	,  )
								train_parser.set_defaults(func=__lowerCamelCase     )
					@staticmethod
					def    _lowerCamelCase (  __lowerCamelCase:				int	,  __lowerCamelCase:				Tuple     )    ->				List[Any]:
								__UpperCAmelCase						:			Optional[int]         =   set(pt_outputs.keys()     )
								__UpperCAmelCase						:			Dict         =   set(tf_outputs.keys()     )
								if pt_out_attrs != tf_out_attrs:
											raise ValueError(
											    f'''The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:'''
											    f''' {tf_out_attrs})'''     )

								# 2. For each output attribute, computes the difference
								def _find_pt_tf_differences(__lowerCamelCase:				Optional[int]	,  __lowerCamelCase:				Any	,  __lowerCamelCase:				int	,  __lowerCamelCase:				List[Any]=""     ):
											# If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
											# recursivelly, keeping the name of the attribute.
											if isinstance(__lowerCamelCase	,  torch.Tensor     ):
														__UpperCAmelCase						:			int         =   np.max(np.abs(pt_out.numpy() - tf_out.numpy()     )     )
														__UpperCAmelCase						:			Union[str, Any]         =   tensor_difference
											else:
														__UpperCAmelCase						:			List[Any]         =   attr_name
														for i, pt_item in enumerate(__lowerCamelCase     ):
																	# If it is a named attribute, we keep the name. Otherwise, just its index.
																	if isinstance(__lowerCamelCase	,  __lowerCamelCase     ):
																				__UpperCAmelCase						:			str         =   root_name + pt_item
																				__UpperCAmelCase						:			Dict         =   tf_out[pt_item]
																				__UpperCAmelCase						:			Optional[int]         =   pt_out[pt_item]
																	else:
																				__UpperCAmelCase						:			str         =   root_name + f'''[{i}]'''
																				__UpperCAmelCase						:			Optional[int]         =   tf_out[i]
																	__UpperCAmelCase						:			Tuple         =   _find_pt_tf_differences(__lowerCamelCase	,  __lowerCamelCase	,  __lowerCamelCase	,  __lowerCamelCase     )

											return differences

								return _find_pt_tf_differences(__lowerCamelCase	,  __lowerCamelCase	,  {}     )
					def __init__(  self:				Optional[int]	,  __lowerCamelCase:				Optional[Any]	,  __lowerCamelCase:				int	,  __lowerCamelCase:				str	,  __lowerCamelCase:				List[Any]	,  __lowerCamelCase:				str	,  __lowerCamelCase:				Union[str, Any]	,  __lowerCamelCase:				Dict	,  __lowerCamelCase:				Any	,  *__lowerCamelCase:				Tuple	,  )    ->				Optional[Any]:
								__UpperCAmelCase						:			Optional[Any]         =   logging.get_logger("transformers-cli/pt_to_tf"     )
								__UpperCAmelCase						:			Optional[Any]         =   model_name
								__UpperCAmelCase						:			Any         =   local_dir if local_dir else os.path.join("/tmp"	,  __lowerCamelCase     )
								__UpperCAmelCase						:			Any         =   max_error
								__UpperCAmelCase						:			Optional[Any]         =   new_weights
								__UpperCAmelCase						:			str         =   no_pr
								__UpperCAmelCase						:			List[Any]         =   push
								__UpperCAmelCase						:			Union[str, Any]         =   extra_commit_description
								__UpperCAmelCase						:			int         =   override_model_class
					def    _lowerCamelCase (  self:				Tuple	,  __lowerCamelCase:				Tuple	,  __lowerCamelCase:				List[str]	,  __lowerCamelCase:				Tuple     )    ->				Any:

								def _get_audio_input():
											__UpperCAmelCase						:			List[Any]         =   load_dataset("hf-internal-testing/librispeech_asr_dummy"	,  "clean"	,  split="validation"     )
											__UpperCAmelCase						:			Tuple         =   ds.sort("id"     ).select(range(2     )     )[:2]["audio"]
											__UpperCAmelCase						:			List[Any]         =   [x["array"] for x in speech_samples]
											return raw_samples

								__UpperCAmelCase						:			Union[str, Any]         =   type(pt_model.config     )
								if model_config_class in PROCESSOR_MAPPING:
											__UpperCAmelCase						:			List[str]         =   AutoProcessor.from_pretrained(self._local_dir     )
											if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
														__UpperCAmelCase						:			Tuple         =   processor.tokenizer.eos_token
								elif model_config_class in IMAGE_PROCESSOR_MAPPING:
											__UpperCAmelCase						:			List[str]         =   AutoImageProcessor.from_pretrained(self._local_dir     )
								elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
											__UpperCAmelCase						:			Tuple         =   AutoFeatureExtractor.from_pretrained(self._local_dir     )
								elif model_config_class in TOKENIZER_MAPPING:
											__UpperCAmelCase						:			Tuple         =   AutoTokenizer.from_pretrained(self._local_dir     )
											if processor.pad_token is None:
														__UpperCAmelCase						:			List[Any]         =   processor.eos_token
								else:
											raise ValueError(f'''Unknown data processing type (model config type: {model_config_class})'''     )

								__UpperCAmelCase						:			Optional[Any]         =   set(inspect.signature(pt_model.forward     ).parameters.keys()     )
								__UpperCAmelCase						:			Optional[int]         =   {}
								if "input_ids" in model_forward_signature:
											processor_inputs.update(
											    {
											        "text": ["Hi there!", "I am a batch with more than one row and different input lengths."],
											        "padding": True,
											        "truncation": True,
											    }     )
								if "pixel_values" in model_forward_signature:
											__UpperCAmelCase						:			Optional[Any]         =   load_dataset("cifar10"	,  "plain_text"	,  split="test"     )[:2]["img"]
											processor_inputs.update({"images": sample_images}     )
								if "input_features" in model_forward_signature:
											__UpperCAmelCase						:			Optional[int]         =   inspect.signature(processor.feature_extractor     ).parameters
											# Pad to the largest input length by default but take feature extractor default
											# padding value if it exists e.g. "max_length" and is not False or None
											if "padding" in feature_extractor_signature:
														__UpperCAmelCase						:			List[Any]         =   feature_extractor_signature["padding"].default
														if default_strategy is not False and default_strategy is not None:
																	__UpperCAmelCase						:			Optional[Any]         =   default_strategy
														else:
																	__UpperCAmelCase						:			Union[str, Any]         =   True
											else:
														__UpperCAmelCase						:			int         =   True
											processor_inputs.update({"audio": _get_audio_input(), "padding": padding_strategy}     )
								if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
											processor_inputs.update({"audio": _get_audio_input(), "padding": True}     )
								__UpperCAmelCase						:			int         =   processor(**__lowerCamelCase	,  return_tensors="pt"     )
								__UpperCAmelCase						:			int         =   processor(**__lowerCamelCase	,  return_tensors="tf"     )

								# Extra input requirements, in addition to the input modality
								if (
								    config.is_encoder_decoder
								    or (hasattr(__lowerCamelCase	,  "encoder"     ) and hasattr(__lowerCamelCase	,  "decoder"     ))
								    or "decoder_input_ids" in tf_dummy_inputs
								):
											__UpperCAmelCase						:			List[str]         =   np.asarray([[1], [1]]	,  dtype=__lowerCamelCase     ) * (pt_model.config.decoder_start_token_id or 0)
											pt_input.update({"decoder_input_ids": torch.tensor(__lowerCamelCase     )}     )
											tf_input.update({"decoder_input_ids": tf.convert_to_tensor(__lowerCamelCase     )}     )

								return pt_input, tf_input





					def    _lowerCamelCase (  self:				Tuple     )    ->				Tuple:
								# hub version 0.9.0 introduced the possibility of programmatically opening PRs with normal write tokens.
								if version.parse(huggingface_hub.__version__     ) < version.parse("0.9.0"     ):
											raise ImportError(
											    "The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"
											    " installation."     )
								else:
											from huggingface_hub import Repository, create_commit
											from huggingface_hub._commit_api import CommitOperationAdd

								# Fetch remote data
								__UpperCAmelCase						:			str         =   Repository(local_dir=self._local_dir	,  clone_from=self._model_name     )

								# Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
								__UpperCAmelCase						:			str         =   AutoConfig.from_pretrained(self._local_dir     )
								__UpperCAmelCase						:			List[Any]         =   config.architectures
								if self._override_model_class is not None:
											if self._override_model_class.startswith("TF"     ):
														__UpperCAmelCase						:			Union[str, Any]         =   [self._override_model_class[2:]]
											else:
														__UpperCAmelCase						:			Tuple         =   [self._override_model_class]
											try:
														__UpperCAmelCase						:			List[str]         =   getattr(import_module("transformers"     )	,  architectures[0]     )
											except AttributeError:
														raise ValueError(f'''Model class {self._override_model_class} not found in transformers.'''     )
											try:
														__UpperCAmelCase						:			Tuple         =   getattr(import_module("transformers"     )	,  "TF" + architectures[0]     )
											except AttributeError:
														raise ValueError(f'''TF model class TF{self._override_model_class} not found in transformers.'''     )
								elif architectures is None:  # No architecture defined -- use auto classes
											__UpperCAmelCase						:			Optional[Any]         =   getattr(import_module("transformers"     )	,  "AutoModel"     )
											__UpperCAmelCase						:			int         =   getattr(import_module("transformers"     )	,  "TFAutoModel"     )
											self._logger.warning("No detected architecture, using AutoModel/TFAutoModel"     )
								else:  # Architecture defined -- use it
											if len(__lowerCamelCase     ) > 1:
														raise ValueError(f'''More than one architecture was found, aborting. (architectures = {architectures})'''     )
											self._logger.warning(f'''Detected architecture: {architectures[0]}'''     )
											__UpperCAmelCase						:			List[Any]         =   getattr(import_module("transformers"     )	,  architectures[0]     )
											try:
														__UpperCAmelCase						:			Tuple         =   getattr(import_module("transformers"     )	,  "TF" + architectures[0]     )
											except AttributeError:
														raise AttributeError(f'''The TensorFlow equivalent of {architectures[0]} doesn\'t exist in transformers.'''     )

														    # Check the TF dummy inputs to see what keys we need in the forward pass
								__UpperCAmelCase						:			int         =   tf_class.from_config(__lowerCamelCase     )
								__UpperCAmelCase						:			str         =   tf_from_pt_model.dummy_inputs

								del tf_from_pt_model  # Try to keep only one model in memory at a time

								# Load the model and get some basic inputs
								__UpperCAmelCase						:			Tuple         =   pt_class.from_pretrained(self._local_dir     )
								pt_model.eval()

								__UpperCAmelCase    ,	__UpperCAmelCase						:			Optional[Any]         =   self.get_inputs(__lowerCamelCase	,  __lowerCamelCase	,  __lowerCamelCase     )

								with torch.no_grad():
											__UpperCAmelCase						:			int         =   pt_model(**__lowerCamelCase	,  output_hidden_states=__lowerCamelCase     )
								del pt_model  # will no longer be used, and may have a large memory footprint

								__UpperCAmelCase						:			Tuple         =   tf_class.from_pretrained(self._local_dir	,  from_pt=__lowerCamelCase     )
								__UpperCAmelCase						:			Any         =   tf_from_pt_model(**__lowerCamelCase	,  output_hidden_states=__lowerCamelCase	,  training=__lowerCamelCase     )

								# Confirms that cross loading PT weights into TF worked.
								__UpperCAmelCase						:			List[str]         =   self.find_pt_tf_differences(__lowerCamelCase	,  __lowerCamelCase     )
								__UpperCAmelCase						:			Optional[Any]         =   {k: v for k, v in crossload_differences.items() if "hidden" not in k}
								__UpperCAmelCase						:			List[Any]         =   {k: v for k, v in crossload_differences.items() if "hidden" in k}
								if len(__lowerCamelCase     ) == 0 and architectures is not None:
											raise ValueError(
											    f'''Something went wrong -- the config file has architectures ({architectures}), but no model head'''
											    " output was found. All outputs start with 'hidden'"     )
								__UpperCAmelCase						:			Tuple         =   max(output_differences.values()     ) if output_differences else 0.0
								__UpperCAmelCase						:			str         =   max(hidden_differences.values()     )
								if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
											raise ValueError(
											    "The cross-loaded TensorFlow model has different outputs, something went wrong!\n"
											    + f'''\nList of maximum output differences above the threshold ({self._max_error}):\n'''
											    + "\n".join([f'''{k}: {v:.3e}''' for k, v in output_differences.items() if v > self._max_error]     )
											    + f'''\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'''
											    + "\n".join([f'''{k}: {v:.3e}''' for k, v in hidden_differences.items() if v > self._max_error]     )     )

								# Save the weights in a TF format (if needed) and confirms that the results are still good
								__UpperCAmelCase						:			Tuple         =   os.path.join(self._local_dir	,  __lowerCamelCase     )
								__UpperCAmelCase						:			Union[str, Any]         =   os.path.join(self._local_dir	,  __lowerCamelCase     )
								if (not os.path.exists(__lowerCamelCase     ) and not os.path.exists(__lowerCamelCase     )) or self._new_weights:
											tf_from_pt_model.save_pretrained(self._local_dir     )
								del tf_from_pt_model  # will no longer be used, and may have a large memory footprint

								__UpperCAmelCase						:			Optional[Any]         =   tf_class.from_pretrained(self._local_dir     )
								__UpperCAmelCase						:			str         =   tf_model(**__lowerCamelCase	,  output_hidden_states=__lowerCamelCase     )

								__UpperCAmelCase						:			Union[str, Any]         =   self.find_pt_tf_differences(__lowerCamelCase	,  __lowerCamelCase     )
								__UpperCAmelCase						:			Optional[int]         =   {k: v for k, v in conversion_differences.items() if "hidden" not in k}
								__UpperCAmelCase						:			Any         =   {k: v for k, v in conversion_differences.items() if "hidden" in k}
								if len(__lowerCamelCase     ) == 0 and architectures is not None:
											raise ValueError(
											    f'''Something went wrong -- the config file has architectures ({architectures}), but no model head'''
											    " output was found. All outputs start with 'hidden'"     )
								__UpperCAmelCase						:			Optional[int]         =   max(output_differences.values()     ) if output_differences else 0.0
								__UpperCAmelCase						:			Tuple         =   max(hidden_differences.values()     )
								if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
											raise ValueError(
											    "The converted TensorFlow model has different outputs, something went wrong!\n"
											    + f'''\nList of maximum output differences above the threshold ({self._max_error}):\n'''
											    + "\n".join([f'''{k}: {v:.3e}''' for k, v in output_differences.items() if v > self._max_error]     )
											    + f'''\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'''
											    + "\n".join([f'''{k}: {v:.3e}''' for k, v in hidden_differences.items() if v > self._max_error]     )     )

								__UpperCAmelCase						:			str         =   "Update TF weights" if self._new_weights else "Add TF weights"
								if self._push:
											repo.git_add(auto_lfs_track=__lowerCamelCase     )
											repo.git_commit(__lowerCamelCase     )
											repo.git_push(blocking=__lowerCamelCase     )  # this prints a progress bar with the upload
											self._logger.warning(f'''TF weights pushed into {self._model_name}'''     )
								elif not self._no_pr:
											self._logger.warning("Uploading the weights into a new PR..."     )
											__UpperCAmelCase						:			Any         =   (
											    "Model converted by the [`transformers`' `pt_to_tf`"
											    " CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). "
											    "All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n"
											    f'''Maximum crossload output difference={max_crossload_output_diff:.3e}; '''
											    f'''Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n'''
											    f'''Maximum conversion output difference={max_conversion_output_diff:.3e}; '''
											    f'''Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n'''
											)
											if self._max_error > MAX_ERROR:
														commit_descrition += (
														    f'''\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!'''
														)
											if self._extra_commit_description:
														commit_descrition += "\n\n" + self._extra_commit_description

											# sharded model -> adds all related files (index and .h5 shards)
											if os.path.exists(__lowerCamelCase     ):
														__UpperCAmelCase						:			str         =   [
														    CommitOperationAdd(path_in_repo=__lowerCamelCase	,  path_or_fileobj=__lowerCamelCase     )
														]
														for shard_path in tf.io.gfile.glob(self._local_dir + "/tf_model-*.h5"     ):
																	operations += [
																	    CommitOperationAdd(path_in_repo=os.path.basename(__lowerCamelCase     )	,  path_or_fileobj=__lowerCamelCase     )
																	]
											else:
														__UpperCAmelCase						:			List[str]         =   [CommitOperationAdd(path_in_repo=__lowerCamelCase	,  path_or_fileobj=__lowerCamelCase     )]

											__UpperCAmelCase						:			Optional[int]         =   create_commit(
											    repo_id=self._model_name	,  operations=__lowerCamelCase	,  commit_message=__lowerCamelCase	,  commit_description=__lowerCamelCase	,  repo_type="model"	,  create_pr=__lowerCamelCase	,  ).pr_url
											self._logger.warning(f'''PR open in {hub_pr_url}'''     )





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
	from tokenizers import Tokenizer
	from tokenizers.models import WordPiece




class 					_snake_case						(						unittest.TestCase				):
					def    _lowerCamelCase (  self:				str	,  __lowerCamelCase:				int     )    ->				List[Any]:
												__UpperCAmelCase						:			List[str]         =   list(tokenizer_class.max_model_input_sizes.keys()     )
												for model_name in sa_models[:1]:
																			__UpperCAmelCase						:			Any         =   tokenizer_class.from_pretrained(__lowerCamelCase     )
																			self.assertIsNotNone(__lowerCamelCase     )
																			self.assertIsInstance(__lowerCamelCase	,  __lowerCamelCase     )
																			self.assertIsInstance(__lowerCamelCase	,  __lowerCamelCase     )

																			for special_tok in tokenizer.all_special_tokens:
																										self.assertIsInstance(__lowerCamelCase	,  __lowerCamelCase     )
																										__UpperCAmelCase						:			str         =   tokenizer.convert_tokens_to_ids(__lowerCamelCase     )
																										self.assertIsInstance(__lowerCamelCase	,  __lowerCamelCase     )
					def    _lowerCamelCase (  self:				List[Any]	,  __lowerCamelCase:				BatchEncoding	,  __lowerCamelCase:				Optional[Callable] = None     )    ->				Dict:
												__UpperCAmelCase						:			int         =   pickle.dumps(__lowerCamelCase     )
												self.assertIsNotNone(__lowerCamelCase     )

												__UpperCAmelCase						:			Dict         =   pickle.loads(__lowerCamelCase     )

												# Ensure is_fast is correctly restored
												self.assertEqual(be_restored.is_fast	,  be_original.is_fast     )

												# Ensure encodings are potentially correctly restored
												if be_original.is_fast:
																			self.assertIsNotNone(be_restored.encodings     )
												else:
																			self.assertIsNone(be_restored.encodings     )

												# Ensure the keys are the same
												for original_v, restored_v in zip(be_original.values()	,  be_restored.values()     ):
																			if equal_op:
																										self.assertTrue(equal_op(__lowerCamelCase	,  __lowerCamelCase     )     )
																			else:
																										self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
					@slow
					def    _lowerCamelCase (  self:				str     )    ->				Optional[Any]:
												self.check_tokenizer_from_pretrained(__lowerCamelCase     )
					def    _lowerCamelCase (  self:				str     )    ->				Dict:
												self.assertEqual(TensorType("tf"     )	,  TensorType.TENSORFLOW     )
												self.assertEqual(TensorType("pt"     )	,  TensorType.PYTORCH     )
												self.assertEqual(TensorType("np"     )	,  TensorType.NUMPY     )
					@require_tokenizers
					def    _lowerCamelCase (  self:				Union[str, Any]     )    ->				int:
												import numpy as np

												__UpperCAmelCase						:			str         =   BertTokenizer.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			Any         =   BertTokenizerFast.from_pretrained("bert-base-cased"     )

												# Python no tensor
												with self.subTest("BatchEncoding (Python, return_tensors=None)"     ):
																			self.assert_dump_and_restore(tokenizer_p("Small example to encode"     )     )

												with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_p("Small example to encode"	,  return_tensors=TensorType.NUMPY     )	,  np.array_equal     )

												with self.subTest("BatchEncoding (Rust, return_tensors=None)"     ):
																			self.assert_dump_and_restore(tokenizer_r("Small example to encode"     )     )

												with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_r("Small example to encode"	,  return_tensors=TensorType.NUMPY     )	,  np.array_equal     )
					@require_tf
					@require_tokenizers
					def    _lowerCamelCase (  self:				Tuple     )    ->				str:
												import tensorflow as tf

												def tf_array_equals(__lowerCamelCase:				Dict	,  __lowerCamelCase:				Tuple     ):
																			return tf.reduce_all(tf.equal(__lowerCamelCase	,  __lowerCamelCase     )     )

												__UpperCAmelCase						:			List[str]         =   BertTokenizer.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			Optional[int]         =   BertTokenizerFast.from_pretrained("bert-base-cased"     )

												with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_p("Small example to encode"	,  return_tensors=TensorType.TENSORFLOW     )	,  __lowerCamelCase     )

												with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_r("Small example to encode"	,  return_tensors=TensorType.TENSORFLOW     )	,  __lowerCamelCase     )
					@require_torch
					@require_tokenizers
					def    _lowerCamelCase (  self:				Tuple     )    ->				Union[str, Any]:
												import torch

												__UpperCAmelCase						:			Optional[int]         =   BertTokenizer.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			Optional[Any]         =   BertTokenizerFast.from_pretrained("bert-base-cased"     )

												with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_p("Small example to encode"	,  return_tensors=TensorType.PYTORCH     )	,  torch.equal     )

												with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_r("Small example to encode"	,  return_tensors=TensorType.PYTORCH     )	,  torch.equal     )
					@require_tokenizers
					def    _lowerCamelCase (  self:				int     )    ->				Any:
												__UpperCAmelCase						:			Tuple         =   BertTokenizer.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			Tuple         =   BertTokenizerFast.from_pretrained("bert-base-cased"     )

												with self.subTest("Python Tokenizer"     ):
																			self.assertFalse(tokenizer_p("Small example to_encode"     ).is_fast     )

												with self.subTest("Rust Tokenizer"     ):
																			self.assertTrue(tokenizer_r("Small example to_encode"     ).is_fast     )
					@require_tokenizers
					def    _lowerCamelCase (  self:				Union[str, Any]     )    ->				List[str]:
												__UpperCAmelCase						:			Tuple         =   BertTokenizerFast.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			Tuple         =   tokenizer_r(["Test", "\xad", "test"]	,  is_split_into_words=__lowerCamelCase     )

												self.assertEqual(encoded.word_to_tokens(0     )	,  TokenSpan(start=1	,  end=2     )     )
												self.assertEqual(encoded.word_to_tokens(1     )	,  __lowerCamelCase     )
												self.assertEqual(encoded.word_to_tokens(2     )	,  TokenSpan(start=2	,  end=3     )     )
					def    _lowerCamelCase (  self:				List[str]     )    ->				List[Any]:
												__UpperCAmelCase						:			str         =   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
												__UpperCAmelCase						:			str         =   batch.convert_to_tensors(tensor_type="np"     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (2, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (2,)     )
												# test converting the converted
												with CaptureStderr() as cs:
																			__UpperCAmelCase						:			Tuple         =   batch.convert_to_tensors(tensor_type="np"     )
												self.assertFalse(len(cs.err     )	,  msg=f'''should have no warning, but got {cs.err}'''     )

												__UpperCAmelCase						:			Optional[Any]         =   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
												__UpperCAmelCase						:			Any         =   batch.convert_to_tensors(tensor_type="np"	,  prepend_batch_axis=__lowerCamelCase     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (1, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (1,)     )
					@require_torch
					def    _lowerCamelCase (  self:				Dict     )    ->				int:
												__UpperCAmelCase						:			str         =   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
												__UpperCAmelCase						:			Union[str, Any]         =   batch.convert_to_tensors(tensor_type="pt"     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (2, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (2,)     )
												# test converting the converted
												with CaptureStderr() as cs:
																			__UpperCAmelCase						:			Optional[int]         =   batch.convert_to_tensors(tensor_type="pt"     )
												self.assertFalse(len(cs.err     )	,  msg=f'''should have no warning, but got {cs.err}'''     )

												__UpperCAmelCase						:			Optional[Any]         =   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
												__UpperCAmelCase						:			Union[str, Any]         =   batch.convert_to_tensors(tensor_type="pt"	,  prepend_batch_axis=__lowerCamelCase     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (1, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (1,)     )
					@require_tf
					def    _lowerCamelCase (  self:				Any     )    ->				Dict:
												__UpperCAmelCase						:			Any         =   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
												__UpperCAmelCase						:			Union[str, Any]         =   batch.convert_to_tensors(tensor_type="tf"     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (2, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (2,)     )
												# test converting the converted
												with CaptureStderr() as cs:
																			__UpperCAmelCase						:			Tuple         =   batch.convert_to_tensors(tensor_type="tf"     )
												self.assertFalse(len(cs.err     )	,  msg=f'''should have no warning, but got {cs.err}'''     )

												__UpperCAmelCase						:			Tuple         =   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
												__UpperCAmelCase						:			int         =   batch.convert_to_tensors(tensor_type="tf"	,  prepend_batch_axis=__lowerCamelCase     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (1, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (1,)     )
					@require_flax
					def    _lowerCamelCase (  self:				Any     )    ->				Tuple:
												__UpperCAmelCase						:			Optional[Any]         =   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
												__UpperCAmelCase						:			List[Any]         =   batch.convert_to_tensors(tensor_type="jax"     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (2, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (2,)     )
												# test converting the converted
												with CaptureStderr() as cs:
																			__UpperCAmelCase						:			Tuple         =   batch.convert_to_tensors(tensor_type="jax"     )
												self.assertFalse(len(cs.err     )	,  msg=f'''should have no warning, but got {cs.err}'''     )

												__UpperCAmelCase						:			str         =   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
												__UpperCAmelCase						:			str         =   batch.convert_to_tensors(tensor_type="jax"	,  prepend_batch_axis=__lowerCamelCase     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (1, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (1,)     )
					def    _lowerCamelCase (  self:				Optional[int]     )    ->				Optional[int]:
												__UpperCAmelCase						:			List[Any]         =   [{"input_ids": np.array([0, 1, 2]     )}, {"input_ids": np.array([0, 1, 2, 3]     )}]
												__UpperCAmelCase						:			Any         =   BertTokenizer.from_pretrained("bert-base-cased"     )

												__UpperCAmelCase						:			Union[str, Any]         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase     )
												self.assertTrue(isinstance(batch["input_ids"]	,  np.ndarray     )     )
												self.assertEqual(batch["input_ids"].tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
												__UpperCAmelCase						:			Union[str, Any]         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase	,  return_tensors="np"     )
												self.assertTrue(isinstance(batch["input_ids"]	,  np.ndarray     )     )
												self.assertEqual(batch["input_ids"].tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
					@require_torch
					def    _lowerCamelCase (  self:				List[Any]     )    ->				Optional[Any]:
												import torch

												__UpperCAmelCase						:			Any         =   [{"input_ids": torch.tensor([0, 1, 2]     )}, {"input_ids": torch.tensor([0, 1, 2, 3]     )}]
												__UpperCAmelCase						:			Any         =   BertTokenizer.from_pretrained("bert-base-cased"     )

												__UpperCAmelCase						:			str         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase     )
												self.assertTrue(isinstance(batch["input_ids"]	,  torch.Tensor     )     )
												self.assertEqual(batch["input_ids"].tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
												__UpperCAmelCase						:			Union[str, Any]         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase	,  return_tensors="pt"     )
												self.assertTrue(isinstance(batch["input_ids"]	,  torch.Tensor     )     )
												self.assertEqual(batch["input_ids"].tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
					@require_tf
					def    _lowerCamelCase (  self:				int     )    ->				Union[str, Any]:
												import tensorflow as tf

												__UpperCAmelCase						:			Any         =   [{"input_ids": tf.constant([0, 1, 2]     )}, {"input_ids": tf.constant([0, 1, 2, 3]     )}]
												__UpperCAmelCase						:			List[str]         =   BertTokenizer.from_pretrained("bert-base-cased"     )

												__UpperCAmelCase						:			str         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase     )
												self.assertTrue(isinstance(batch["input_ids"]	,  tf.Tensor     )     )
												self.assertEqual(batch["input_ids"].numpy().tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
												__UpperCAmelCase						:			Dict         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase	,  return_tensors="tf"     )
												self.assertTrue(isinstance(batch["input_ids"]	,  tf.Tensor     )     )
												self.assertEqual(batch["input_ids"].numpy().tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
					@require_tokenizers
					def    _lowerCamelCase (  self:				Dict     )    ->				int:
												__UpperCAmelCase						:			Optional[int]         =   Tokenizer(WordPiece(unk_token="[UNK]"     )     )
												PreTrainedTokenizerFast(tokenizer_object=__lowerCamelCase     )





					@require_tokenizers
					def    _lowerCamelCase (  self:				str     )    ->				Dict:
												__UpperCAmelCase						:			Tuple         =   Tokenizer(WordPiece(unk_token="[UNK]"     )     )
												with tempfile.TemporaryDirectory() as tmpdirname:
																			bert_tokenizer.save(os.path.join(__lowerCamelCase	,  "tokenizer.json"     )     )
																			PreTrainedTokenizerFast(tokenizer_file=os.path.join(__lowerCamelCase	,  "tokenizer.json"     )     )




