
"""simple docstring"""






class  lowercase				:






   def __init__(self     :  int )					->  List[str]:





    """simple docstring"""


    lowerCAmelCase				       =     []
    lowerCAmelCase				       =     0
    lowerCAmelCase				       =     0






   def __str__(self     :  Optional[int] )					->  Optional[int]:





    """simple docstring"""


    lowerCAmelCase				       =     '''<''' + str(self.entries )[1:-1] + '''>'''
    return printed






   def 				UpperCAmelCase       (self     :  Any			,SCREAMING_SNAKE_CASE_     :  Tuple )					->  Dict:





    """simple docstring"""


    self.entries.append(SCREAMING_SNAKE_CASE_ )
    lowerCAmelCase				       =     self.length + 1






   def 				UpperCAmelCase       (self     :  Any )					->  Union[str, Any]:





    """simple docstring"""


    lowerCAmelCase				       =     self.length - 1
    lowerCAmelCase				       =     self.entries[self.front]
    # self.front-=1
    # self.entries = self.entries[self.front:]
    lowerCAmelCase				       =     self.entries[1:]
    return dequeued






   def 				UpperCAmelCase       (self     :  int			,SCREAMING_SNAKE_CASE_     :  List[str] )					->  Tuple:





    """simple docstring"""


    for _ in range(SCREAMING_SNAKE_CASE_ ):
     self.put(self.get() )






   def 				UpperCAmelCase       (self     :  List[Any] )					->  Optional[Any]:





    """simple docstring"""


    return self.entries[0]






   def 				UpperCAmelCase       (self     :  List[str] )					->  Dict:





    """simple docstring"""


    return self.length




"""simple docstring"""






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
					from tokenizers import Tokenizer
					from tokenizers.models import WordPiece



class  lowercase				(   unittest.TestCase		):






			def 				UpperCAmelCase       (self     :  str			,SCREAMING_SNAKE_CASE_     :  Dict )					->  int:





				"""simple docstring"""


				lowerCAmelCase				       =     list(tokenizer_class.max_model_input_sizes.keys() )
				for model_name in sa_models[:1]:
					lowerCAmelCase				       =     tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ )
					self.assertIsNotNone(SCREAMING_SNAKE_CASE_ )
					self.assertIsInstance(SCREAMING_SNAKE_CASE_			,SCREAMING_SNAKE_CASE_ )
					self.assertIsInstance(SCREAMING_SNAKE_CASE_			,SCREAMING_SNAKE_CASE_ )

					for special_tok in tokenizer.all_special_tokens:
						self.assertIsInstance(SCREAMING_SNAKE_CASE_			,SCREAMING_SNAKE_CASE_ )
						lowerCAmelCase				       =     tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ )
						self.assertIsInstance(SCREAMING_SNAKE_CASE_			,SCREAMING_SNAKE_CASE_ )






			def 				UpperCAmelCase       (self     :  str			,SCREAMING_SNAKE_CASE_     :  BatchEncoding			,SCREAMING_SNAKE_CASE_     :  Optional[Callable] = None )					->  Dict:





				"""simple docstring"""


				lowerCAmelCase				       =     pickle.dumps(SCREAMING_SNAKE_CASE_ )
				self.assertIsNotNone(SCREAMING_SNAKE_CASE_ )

				lowerCAmelCase				       =     pickle.loads(SCREAMING_SNAKE_CASE_ )

				# Ensure is_fast is correctly restored
				self.assertEqual(be_restored.is_fast			,be_original.is_fast )

				# Ensure encodings are potentially correctly restored
				if be_original.is_fast:
					self.assertIsNotNone(be_restored.encodings )
				else:
					self.assertIsNone(be_restored.encodings )

				# Ensure the keys are the same
				for original_v, restored_v in zip(be_original.values()			,be_restored.values() ):
					if equal_op:
						self.assertTrue(equal_op(SCREAMING_SNAKE_CASE_			,SCREAMING_SNAKE_CASE_ ) )
					else:
						self.assertEqual(SCREAMING_SNAKE_CASE_			,SCREAMING_SNAKE_CASE_ )






			@slow
			def 				UpperCAmelCase       (self     :  List[str] )					->  int:





				"""simple docstring"""


				self.check_tokenizer_from_pretrained(SCREAMING_SNAKE_CASE_ )






			def 				UpperCAmelCase       (self     :  str )					->  Dict:





				"""simple docstring"""


				self.assertEqual(TensorType('''tf''' )			,TensorType.TENSORFLOW )
				self.assertEqual(TensorType('''pt''' )			,TensorType.PYTORCH )
				self.assertEqual(TensorType('''np''' )			,TensorType.NUMPY )






			@require_tokenizers
			def 				UpperCAmelCase       (self     :  int )					->  Tuple:





				"""simple docstring"""


				import numpy as np

				lowerCAmelCase				       =     BertTokenizer.from_pretrained('''bert-base-cased''' )
				lowerCAmelCase				       =     BertTokenizerFast.from_pretrained('''bert-base-cased''' )

				# Python no tensor
				with self.subTest('''BatchEncoding (Python, return_tensors=None)''' ):
					self.assert_dump_and_restore(tokenizer_p('''Small example to encode''' ) )

				with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)''' ):
					self.assert_dump_and_restore(
					    tokenizer_p('''Small example to encode'''			,return_tensors=TensorType.NUMPY )			,np.array_equal )

				with self.subTest('''BatchEncoding (Rust, return_tensors=None)''' ):
					self.assert_dump_and_restore(tokenizer_r('''Small example to encode''' ) )

				with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)''' ):
					self.assert_dump_and_restore(
					    tokenizer_r('''Small example to encode'''			,return_tensors=TensorType.NUMPY )			,np.array_equal )






			@require_tf
			@require_tokenizers
			def 				UpperCAmelCase       (self     :  str )					->  int:





				"""simple docstring"""


				import tensorflow as tf

				def tf_array_equals(SCREAMING_SNAKE_CASE_     :  Optional[Any]			,SCREAMING_SNAKE_CASE_     :  List[Any] ):
					return tf.reduce_all(tf.equal(SCREAMING_SNAKE_CASE_			,SCREAMING_SNAKE_CASE_ ) )

				lowerCAmelCase				       =     BertTokenizer.from_pretrained('''bert-base-cased''' )
				lowerCAmelCase				       =     BertTokenizerFast.from_pretrained('''bert-base-cased''' )

				with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)''' ):
					self.assert_dump_and_restore(
					    tokenizer_p('''Small example to encode'''			,return_tensors=TensorType.TENSORFLOW )			,SCREAMING_SNAKE_CASE_ )

				with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)''' ):
					self.assert_dump_and_restore(
					    tokenizer_r('''Small example to encode'''			,return_tensors=TensorType.TENSORFLOW )			,SCREAMING_SNAKE_CASE_ )






			@require_torch
			@require_tokenizers
			def 				UpperCAmelCase       (self     :  Optional[Any] )					->  Union[str, Any]:





				"""simple docstring"""


				import torch

				lowerCAmelCase				       =     BertTokenizer.from_pretrained('''bert-base-cased''' )
				lowerCAmelCase				       =     BertTokenizerFast.from_pretrained('''bert-base-cased''' )

				with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)''' ):
					self.assert_dump_and_restore(
					    tokenizer_p('''Small example to encode'''			,return_tensors=TensorType.PYTORCH )			,torch.equal )

				with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)''' ):
					self.assert_dump_and_restore(
					    tokenizer_r('''Small example to encode'''			,return_tensors=TensorType.PYTORCH )			,torch.equal )






			@require_tokenizers
			def 				UpperCAmelCase       (self     :  str )					->  Tuple:





				"""simple docstring"""


				lowerCAmelCase				       =     BertTokenizer.from_pretrained('''bert-base-cased''' )
				lowerCAmelCase				       =     BertTokenizerFast.from_pretrained('''bert-base-cased''' )

				with self.subTest('''Python Tokenizer''' ):
					self.assertFalse(tokenizer_p('''Small example to_encode''' ).is_fast )

				with self.subTest('''Rust Tokenizer''' ):
					self.assertTrue(tokenizer_r('''Small example to_encode''' ).is_fast )






			@require_tokenizers
			def 				UpperCAmelCase       (self     :  Any )					->  Dict:





				"""simple docstring"""


				lowerCAmelCase				       =     BertTokenizerFast.from_pretrained('''bert-base-cased''' )
				lowerCAmelCase				       =     tokenizer_r(['''Test''', '''\xad''', '''test''']			,is_split_into_words=SCREAMING_SNAKE_CASE_ )

				self.assertEqual(encoded.word_to_tokens(0 )			,TokenSpan(start=1			,end=2 ) )
				self.assertEqual(encoded.word_to_tokens(1 )			,SCREAMING_SNAKE_CASE_ )
				self.assertEqual(encoded.word_to_tokens(2 )			,TokenSpan(start=2			,end=3 ) )






			def 				UpperCAmelCase       (self     :  List[Any] )					->  int:





				"""simple docstring"""


				lowerCAmelCase				       =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
				lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''np''' )
				self.assertEqual(tensor_batch['''inputs'''].shape			,(2, 3) )
				self.assertEqual(tensor_batch['''labels'''].shape			,(2,) )
				# test converting the converted
				with CaptureStderr() as cs:
					lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''np''' )
				self.assertFalse(len(cs.err )			,msg=F"""should have no warning, but got {cs.err}""" )

				lowerCAmelCase				       =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
				lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''np'''			,prepend_batch_axis=SCREAMING_SNAKE_CASE_ )
				self.assertEqual(tensor_batch['''inputs'''].shape			,(1, 3) )
				self.assertEqual(tensor_batch['''labels'''].shape			,(1,) )






			@require_torch
			def 				UpperCAmelCase       (self     :  Union[str, Any] )					->  Optional[int]:





				"""simple docstring"""


				lowerCAmelCase				       =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
				lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''pt''' )
				self.assertEqual(tensor_batch['''inputs'''].shape			,(2, 3) )
				self.assertEqual(tensor_batch['''labels'''].shape			,(2,) )
				# test converting the converted
				with CaptureStderr() as cs:
					lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''pt''' )
				self.assertFalse(len(cs.err )			,msg=F"""should have no warning, but got {cs.err}""" )

				lowerCAmelCase				       =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
				lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''pt'''			,prepend_batch_axis=SCREAMING_SNAKE_CASE_ )
				self.assertEqual(tensor_batch['''inputs'''].shape			,(1, 3) )
				self.assertEqual(tensor_batch['''labels'''].shape			,(1,) )






			@require_tf
			def 				UpperCAmelCase       (self     :  List[Any] )					->  List[Any]:





				"""simple docstring"""


				lowerCAmelCase				       =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
				lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''tf''' )
				self.assertEqual(tensor_batch['''inputs'''].shape			,(2, 3) )
				self.assertEqual(tensor_batch['''labels'''].shape			,(2,) )
				# test converting the converted
				with CaptureStderr() as cs:
					lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''tf''' )
				self.assertFalse(len(cs.err )			,msg=F"""should have no warning, but got {cs.err}""" )

				lowerCAmelCase				       =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
				lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''tf'''			,prepend_batch_axis=SCREAMING_SNAKE_CASE_ )
				self.assertEqual(tensor_batch['''inputs'''].shape			,(1, 3) )
				self.assertEqual(tensor_batch['''labels'''].shape			,(1,) )






			@require_flax
			def 				UpperCAmelCase       (self     :  Union[str, Any] )					->  Optional[int]:





				"""simple docstring"""


				lowerCAmelCase				       =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
				lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''jax''' )
				self.assertEqual(tensor_batch['''inputs'''].shape			,(2, 3) )
				self.assertEqual(tensor_batch['''labels'''].shape			,(2,) )
				# test converting the converted
				with CaptureStderr() as cs:
					lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''jax''' )
				self.assertFalse(len(cs.err )			,msg=F"""should have no warning, but got {cs.err}""" )

				lowerCAmelCase				       =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
				lowerCAmelCase				       =     batch.convert_to_tensors(tensor_type='''jax'''			,prepend_batch_axis=SCREAMING_SNAKE_CASE_ )
				self.assertEqual(tensor_batch['''inputs'''].shape			,(1, 3) )
				self.assertEqual(tensor_batch['''labels'''].shape			,(1,) )






			def 				UpperCAmelCase       (self     :  Union[str, Any] )					->  Optional[Any]:





				"""simple docstring"""


				lowerCAmelCase				       =     [{'''input_ids''': np.array([0, 1, 2] )}, {'''input_ids''': np.array([0, 1, 2, 3] )}]
				lowerCAmelCase				       =     BertTokenizer.from_pretrained('''bert-base-cased''' )

				lowerCAmelCase				       =     tokenizer.pad(SCREAMING_SNAKE_CASE_			,padding=SCREAMING_SNAKE_CASE_ )
				self.assertTrue(isinstance(batch['''input_ids''']			,np.ndarray ) )
				self.assertEqual(batch['''input_ids'''].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
				lowerCAmelCase				       =     tokenizer.pad(SCREAMING_SNAKE_CASE_			,padding=SCREAMING_SNAKE_CASE_			,return_tensors='''np''' )
				self.assertTrue(isinstance(batch['''input_ids''']			,np.ndarray ) )
				self.assertEqual(batch['''input_ids'''].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )






			@require_torch
			def 				UpperCAmelCase       (self     :  str )					->  Tuple:





				"""simple docstring"""


				import torch

				lowerCAmelCase				       =     [{'''input_ids''': torch.tensor([0, 1, 2] )}, {'''input_ids''': torch.tensor([0, 1, 2, 3] )}]
				lowerCAmelCase				       =     BertTokenizer.from_pretrained('''bert-base-cased''' )

				lowerCAmelCase				       =     tokenizer.pad(SCREAMING_SNAKE_CASE_			,padding=SCREAMING_SNAKE_CASE_ )
				self.assertTrue(isinstance(batch['''input_ids''']			,torch.Tensor ) )
				self.assertEqual(batch['''input_ids'''].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
				lowerCAmelCase				       =     tokenizer.pad(SCREAMING_SNAKE_CASE_			,padding=SCREAMING_SNAKE_CASE_			,return_tensors='''pt''' )
				self.assertTrue(isinstance(batch['''input_ids''']			,torch.Tensor ) )
				self.assertEqual(batch['''input_ids'''].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )






			@require_tf
			def 				UpperCAmelCase       (self     :  Tuple )					->  Any:





				"""simple docstring"""


				import tensorflow as tf

				lowerCAmelCase				       =     [{'''input_ids''': tf.constant([0, 1, 2] )}, {'''input_ids''': tf.constant([0, 1, 2, 3] )}]
				lowerCAmelCase				       =     BertTokenizer.from_pretrained('''bert-base-cased''' )

				lowerCAmelCase				       =     tokenizer.pad(SCREAMING_SNAKE_CASE_			,padding=SCREAMING_SNAKE_CASE_ )
				self.assertTrue(isinstance(batch['''input_ids''']			,tf.Tensor ) )
				self.assertEqual(batch['''input_ids'''].numpy().tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
				lowerCAmelCase				       =     tokenizer.pad(SCREAMING_SNAKE_CASE_			,padding=SCREAMING_SNAKE_CASE_			,return_tensors='''tf''' )
				self.assertTrue(isinstance(batch['''input_ids''']			,tf.Tensor ) )
				self.assertEqual(batch['''input_ids'''].numpy().tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )






			@require_tokenizers
			def 				UpperCAmelCase       (self     :  Union[str, Any] )					->  List[str]:





				"""simple docstring"""


				lowerCAmelCase				       =     Tokenizer(WordPiece(unk_token='''[UNK]''' ) )
				PreTrainedTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE_ )






			@require_tokenizers
			def 				UpperCAmelCase       (self     :  Tuple )					->  Optional[Any]:





				"""simple docstring"""


				lowerCAmelCase				       =     Tokenizer(WordPiece(unk_token='''[UNK]''' ) )
				with tempfile.TemporaryDirectory() as tmpdirname:
					bert_tokenizer.save(os.path.join(SCREAMING_SNAKE_CASE_			,'''tokenizer.json''' ) )
					PreTrainedTokenizerFast(tokenizer_file=os.path.join(SCREAMING_SNAKE_CASE_			,'''tokenizer.json''' ) )


