




from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


__lowerCAmelCase						:							List[Any]			     =						logging.get_logger(__name__)


if is_vision_available():
		import PIL




class        snake_case__				(lowercase__  ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			:				Union[str, Any]									=   ["""pixel_values"""]





				def __init__( self							:	int   , __lowerCamelCase							:	int = True   , __lowerCamelCase							:	Optional[int] = None   , __lowerCamelCase							:	Union[str, Any] = PILImageResampling.BICUBIC   , __lowerCamelCase							:	Optional[int] = True   , __lowerCamelCase							:	List[str] = None   , __lowerCamelCase							:	Optional[Any] = True   , __lowerCamelCase							:	List[str] = 1 / 2_55   , __lowerCamelCase							:	List[Any] = True   , __lowerCamelCase							:	List[str] = None   , __lowerCamelCase							:	str = None   , __lowerCamelCase							:	Dict = True   , **__lowerCamelCase							:	int   , )   ->	int:
							super().__init__(**SCREAMING_SNAKE_CASE__				)
							a       =     size if size is not None else {"""shortest_edge""": 2_24}
							a       =     get_size_dict(SCREAMING_SNAKE_CASE__   , default_to_square=SCREAMING_SNAKE_CASE__				)
							a       =     crop_size if crop_size is not None else {"""height""": 2_24, """width""": 2_24}
							a       =     get_size_dict(SCREAMING_SNAKE_CASE__				)

							a       =     do_resize
							a       =     size
							a       =     resample
							a       =     do_center_crop
							a       =     crop_size
							a       =     do_rescale
							a       =     rescale_factor
							a       =     do_normalize
							a       =     image_mean if image_mean is not None else OPENAI_CLIP_MEAN
							a       =     image_std if image_std is not None else OPENAI_CLIP_STD
							a       =     do_convert_rgb





				def 				__UpperCAmelCase				( self							:	Optional[Any]   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	str   , __lowerCamelCase							:	str = PILImageResampling.BICUBIC   , __lowerCamelCase							:	Optional[int] = None   , **__lowerCamelCase							:	Any   , )   ->	Union[str, Any]:
							a       =     get_size_dict(SCREAMING_SNAKE_CASE__   , default_to_square=SCREAMING_SNAKE_CASE__				)
							a       =     get_resize_output_image_size(
							    SCREAMING_SNAKE_CASE__   , size=(size["height"], size["width"])   , default_to_square=SCREAMING_SNAKE_CASE__				)
							return resize(SCREAMING_SNAKE_CASE__   , size=SCREAMING_SNAKE_CASE__   , resample=SCREAMING_SNAKE_CASE__   , data_format=SCREAMING_SNAKE_CASE__   , **SCREAMING_SNAKE_CASE__				)





				def 				__UpperCAmelCase				( self							:	Dict   , __lowerCamelCase							:	str   , __lowerCamelCase							:	str   , __lowerCamelCase							:	str = None   , **__lowerCamelCase							:	Any   , )   ->	Optional[int]:
							a       =     get_size_dict(SCREAMING_SNAKE_CASE__				)
							return center_crop(SCREAMING_SNAKE_CASE__   , size=(size["height"], size["width"])   , data_format=SCREAMING_SNAKE_CASE__   , **SCREAMING_SNAKE_CASE__				)





				def 				__UpperCAmelCase				( self							:	Optional[int]   , __lowerCamelCase							:	int   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	Tuple = None   , **__lowerCamelCase							:	Union[str, Any]   , )   ->	Optional[int]:
							return rescale(SCREAMING_SNAKE_CASE__   , scale=SCREAMING_SNAKE_CASE__   , data_format=SCREAMING_SNAKE_CASE__   , **SCREAMING_SNAKE_CASE__				)





				def 				__UpperCAmelCase				( self							:	Dict   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Tuple = None   , **__lowerCamelCase							:	Optional[Any]   , )   ->	Optional[int]:
							return normalize(SCREAMING_SNAKE_CASE__   , mean=SCREAMING_SNAKE_CASE__   , std=SCREAMING_SNAKE_CASE__   , data_format=SCREAMING_SNAKE_CASE__   , **SCREAMING_SNAKE_CASE__				)





				def 				__UpperCAmelCase				( self							:	str   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Tuple = None   , __lowerCamelCase							:	Optional[int] = None   , __lowerCamelCase							:	List[str] = None   , __lowerCamelCase							:	Optional[int] = None   , __lowerCamelCase							:	int = None   , __lowerCamelCase							:	List[str] = None   , __lowerCamelCase							:	str = None   , __lowerCamelCase							:	List[str] = None   , __lowerCamelCase							:	str = None   , __lowerCamelCase							:	Tuple = None   , __lowerCamelCase							:	Dict = None   , __lowerCamelCase							:	int = None   , __lowerCamelCase							:	Dict = ChannelDimension.FIRST   , **__lowerCamelCase							:	Optional[Any]   , )   ->	Union[str, Any]:
							a       =     do_resize if do_resize is not None else self.do_resize
							a       =     size if size is not None else self.size
							a       =     get_size_dict(SCREAMING_SNAKE_CASE__   , default_to_square=SCREAMING_SNAKE_CASE__				)
							a       =     resample if resample is not None else self.resample
							a       =     do_center_crop if do_center_crop is not None else self.do_center_crop
							a       =     crop_size if crop_size is not None else self.crop_size
							a       =     get_size_dict(SCREAMING_SNAKE_CASE__				)
							a       =     do_rescale if do_rescale is not None else self.do_rescale
							a       =     rescale_factor if rescale_factor is not None else self.rescale_factor
							a       =     do_normalize if do_normalize is not None else self.do_normalize
							a       =     image_mean if image_mean is not None else self.image_mean
							a       =     image_std if image_std is not None else self.image_std
							a       =     do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

							a       =     make_list_of_images(SCREAMING_SNAKE_CASE__				)

							if not valid_images(SCREAMING_SNAKE_CASE__				):
										raise ValueError(
										    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
										    "torch.Tensor, tf.Tensor or jax.ndarray."				)

							if do_resize and size is None:
										raise ValueError("Size must be specified if do_resize is True."				)

							if do_center_crop and crop_size is None:
										raise ValueError("Crop size must be specified if do_center_crop is True."				)

							if do_rescale and rescale_factor is None:
										raise ValueError("Rescale factor must be specified if do_rescale is True."				)

							if do_normalize and (image_mean is None or image_std is None):
										raise ValueError("Image mean and std must be specified if do_normalize is True."				)

							# PIL RGBA images are converted to RGB
							if do_convert_rgb:
										a       =     [convert_to_rgb(SCREAMING_SNAKE_CASE__				) for image in images]

							# All transformations expect numpy arrays.
							a       =     [to_numpy_array(SCREAMING_SNAKE_CASE__				) for image in images]

							if do_resize:
										a       =     [self.resize(image=SCREAMING_SNAKE_CASE__   , size=SCREAMING_SNAKE_CASE__   , resample=SCREAMING_SNAKE_CASE__				) for image in images]

							if do_center_crop:
										a       =     [self.center_crop(image=SCREAMING_SNAKE_CASE__   , size=SCREAMING_SNAKE_CASE__				) for image in images]

							if do_rescale:
										a       =     [self.rescale(image=SCREAMING_SNAKE_CASE__   , scale=SCREAMING_SNAKE_CASE__				) for image in images]

							if do_normalize:
										a       =     [self.normalize(image=SCREAMING_SNAKE_CASE__   , mean=SCREAMING_SNAKE_CASE__   , std=SCREAMING_SNAKE_CASE__				) for image in images]

							a       =     [to_channel_dimension_format(SCREAMING_SNAKE_CASE__   , SCREAMING_SNAKE_CASE__				) for image in images]

							a       =     {"""pixel_values""": images}
							return BatchFeature(data=SCREAMING_SNAKE_CASE__   , tensor_type=SCREAMING_SNAKE_CASE__				)











import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece





class 		__SCREAMING_SNAKE_CASE      (				unittest.TestCase      ):






				"""simple docstring"""

				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__      ):






					"""simple docstring"""

					_snake_case					:							Any					     =   list(tokenizer_class.max_model_input_sizes.keys()      )
					for model_name in sa_models[:1]:
						_snake_case					:							Optional[Any]					     =   tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__      )
						self.assertIsNotNone(SCREAMING_SNAKE_CASE__      )
						self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )
						self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )

						for special_tok in tokenizer.all_special_tokens:
							self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )
							_snake_case					:							int					     =   tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__      )
							self.assertIsInstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None      ):






					"""simple docstring"""

					_snake_case					:							int					     =   pickle.dumps(SCREAMING_SNAKE_CASE__      )
					self.assertIsNotNone(SCREAMING_SNAKE_CASE__      )

					_snake_case					:							Dict					     =   pickle.loads(SCREAMING_SNAKE_CASE__      )

					# Ensure is_fast is correctly restored
					self.assertEqual(be_restored.is_fast						,	be_original.is_fast      )

					# Ensure encodings are potentially correctly restored
					if be_original.is_fast:
						self.assertIsNotNone(be_restored.encodings      )
					else:
						self.assertIsNone(be_restored.encodings      )

					# Ensure the keys are the same
					for original_v, restored_v in zip(be_original.values()						,	be_restored.values()      ):
						if equal_op:
							self.assertTrue(equal_op(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )      )
						else:
							self.assertEqual(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )







				@slow
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					self.check_tokenizer_from_pretrained(SCREAMING_SNAKE_CASE__      )







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					self.assertEqual(TensorType("""tf"""      )						,	TensorType.TENSORFLOW      )
					self.assertEqual(TensorType("""pt"""      )						,	TensorType.PYTORCH      )
					self.assertEqual(TensorType("""np"""      )						,	TensorType.NUMPY      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import numpy as np

					_snake_case					:							Optional[int]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							Any					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					# Python no tensor
					with self.subTest("""BatchEncoding (Python, return_tensors=None)"""      ):
						self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""      )      )

					with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_p("""Small example to encode"""						,	return_tensors=TensorType.NUMPY      )						,	np.array_equal      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""      ):
						self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""      )      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_r("""Small example to encode"""						,	return_tensors=TensorType.NUMPY      )						,	np.array_equal      )







				@require_tf
				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import tensorflow as tf

					def tf_array_equals(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      ):
						return tf.reduce_all(tf.equal(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )      )

					_snake_case					:							int					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							str					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_p("""Small example to encode"""						,	return_tensors=TensorType.TENSORFLOW      )						,	SCREAMING_SNAKE_CASE__      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_r("""Small example to encode"""						,	return_tensors=TensorType.TENSORFLOW      )						,	SCREAMING_SNAKE_CASE__      )







				@require_torch
				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import torch

					_snake_case					:							Union[str, Any]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							Dict					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_p("""Small example to encode"""						,	return_tensors=TensorType.PYTORCH      )						,	torch.equal      )

					with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""      ):
						self.assert_dump_and_restore(
						    tokenizer_r("""Small example to encode"""						,	return_tensors=TensorType.PYTORCH      )						,	torch.equal      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							int					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )

					with self.subTest("""Python Tokenizer"""      ):
						self.assertFalse(tokenizer_p("""Small example to_encode"""      ).is_fast      )

					with self.subTest("""Rust Tokenizer"""      ):
						self.assertTrue(tokenizer_r("""Small example to_encode"""      ).is_fast      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""      )
					_snake_case					:							Dict					     =   tokenizer_r(["""Test""", """\xad""", """test"""]						,	is_split_into_words=SCREAMING_SNAKE_CASE__      )

					self.assertEqual(encoded.word_to_tokens(0      )						,	TokenSpan(start=1						,	end=2      )      )
					self.assertEqual(encoded.word_to_tokens(1      )						,	SCREAMING_SNAKE_CASE__      )
					self.assertEqual(encoded.word_to_tokens(2      )						,	TokenSpan(start=2						,	end=3      )      )







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							List[Any]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							Union[str, Any]					     =   batch.convert_to_tensors(tensor_type="""np"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							Optional[int]					     =   batch.convert_to_tensors(tensor_type="""np"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							str					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							Optional[int]					     =   batch.convert_to_tensors(tensor_type="""np"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				@require_torch
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							List[Any]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							Union[str, Any]					     =   batch.convert_to_tensors(tensor_type="""pt"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							Optional[Any]					     =   batch.convert_to_tensors(tensor_type="""pt"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							Union[str, Any]					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							Union[str, Any]					     =   batch.convert_to_tensors(tensor_type="""pt"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				@require_tf
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							str					     =   batch.convert_to_tensors(tensor_type="""tf"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							int					     =   batch.convert_to_tensors(tensor_type="""tf"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							Optional[Any]					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							List[str]					     =   batch.convert_to_tensors(tensor_type="""tf"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				@require_flax
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							List[Any]					     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}      )
					_snake_case					:							Any					     =   batch.convert_to_tensors(tensor_type="""jax"""      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(2, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(2,)      )
					# test converting the converted
					with CaptureStderr() as cs:
						_snake_case					:							Dict					     =   batch.convert_to_tensors(tensor_type="""jax"""      )
					self.assertFalse(len(cs.err      )						,	msg=f'''should have no warning, but got {cs.err}'''      )

					_snake_case					:							List[str]					     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}      )
					_snake_case					:							List[Any]					     =   batch.convert_to_tensors(tensor_type="""jax"""						,	prepend_batch_axis=SCREAMING_SNAKE_CASE__      )
					self.assertEqual(tensor_batch["""inputs"""].shape						,	(1, 3)      )
					self.assertEqual(tensor_batch["""labels"""].shape						,	(1,)      )







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Any					     =   [{"""input_ids""": np.array([0, 1, 2]      )}, {"""input_ids""": np.array([0, 1, 2, 3]      )}]
					_snake_case					:							int					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )

					_snake_case					:							List[Any]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	np.ndarray      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
					_snake_case					:							Optional[int]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	return_tensors="""np"""      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	np.ndarray      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )







				@require_torch
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import torch

					_snake_case					:							str					     =   [{"""input_ids""": torch.tensor([0, 1, 2]      )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]      )}]
					_snake_case					:							Tuple					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )

					_snake_case					:							Dict					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	torch.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
					_snake_case					:							Tuple					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	return_tensors="""pt"""      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	torch.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )







				@require_tf
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					import tensorflow as tf

					_snake_case					:							Optional[Any]					     =   [{"""input_ids""": tf.constant([0, 1, 2]      )}, {"""input_ids""": tf.constant([0, 1, 2, 3]      )}]
					_snake_case					:							Optional[Any]					     =   BertTokenizer.from_pretrained("""bert-base-cased"""      )

					_snake_case					:							Optional[Any]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	tf.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].numpy().tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
					_snake_case					:							List[Any]					     =   tokenizer.pad(SCREAMING_SNAKE_CASE__						,	padding=SCREAMING_SNAKE_CASE__						,	return_tensors="""tf"""      )
					self.assertTrue(isinstance(batch["""input_ids"""]						,	tf.Tensor      )      )
					self.assertEqual(batch["""input_ids"""].numpy().tolist()						,	[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   Tokenizer(WordPiece(unk_token="""[UNK]"""      )      )
					PreTrainedTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE__      )







				@require_tokenizers
				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					_snake_case					:							Optional[Any]					     =   Tokenizer(WordPiece(unk_token="""[UNK]"""      )      )
					with tempfile.TemporaryDirectory() as tmpdirname:
						bert_tokenizer.save(os.path.join(SCREAMING_SNAKE_CASE__						,	"""tokenizer.json"""      )      )
						PreTrainedTokenizerFast(tokenizer_file=os.path.join(SCREAMING_SNAKE_CASE__						,	"""tokenizer.json"""      )      )





