




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
   from tokenizers import Tokenizer
   from tokenizers.models import WordPiece

class   UpperCAmelCase    (      unittest.TestCase ):






   def 			_a    (						self :							Union[str, Any]    ,       UpperCamelCase_ :							Dict  )->      List[str]:
     _lowercase:			Optional[Any]      =     list(tokenizer_class.max_model_input_sizes.keys()  )
     for model_name in sa_models[:1]:
       _lowercase:			Dict      =     tokenizer_class.from_pretrained(UpperCamelCase_  )
       self.assertIsNotNone(UpperCamelCase_  )
       self.assertIsInstance(UpperCamelCase_    ,       UpperCamelCase_  )
       self.assertIsInstance(UpperCamelCase_    ,       UpperCamelCase_  )

       for special_tok in tokenizer.all_special_tokens:
         self.assertIsInstance(UpperCamelCase_    ,       UpperCamelCase_  )
         _lowercase:			Union[str, Any]      =     tokenizer.convert_tokens_to_ids(UpperCamelCase_  )
         self.assertIsInstance(UpperCamelCase_    ,       UpperCamelCase_  )






   def 			_a    (						self :							Any    ,       UpperCamelCase_ :							BatchEncoding    ,       UpperCamelCase_ :							Optional[Callable] = None  )->      Any:
     _lowercase:			Optional[Any]      =     pickle.dumps(UpperCamelCase_  )
     self.assertIsNotNone(UpperCamelCase_  )

     _lowercase:			Any      =     pickle.loads(UpperCamelCase_  )

     # Ensure is_fast is correctly restored
     self.assertEqual(be_restored.is_fast    ,       be_original.is_fast  )

     # Ensure encodings are potentially correctly restored
     if be_original.is_fast:
       self.assertIsNotNone(be_restored.encodings  )
     else:
       self.assertIsNone(be_restored.encodings  )

     # Ensure the keys are the same
     for original_v, restored_v in zip(be_original.values()    ,       be_restored.values()  ):
       if equal_op:
         self.assertTrue(equal_op(UpperCamelCase_    ,       UpperCamelCase_  )  )
       else:
         self.assertEqual(UpperCamelCase_    ,       UpperCamelCase_  )






   @slow
   def 			_a    (						self :							Dict  )->      Dict:
     self.check_tokenizer_from_pretrained(UpperCamelCase_  )






   def 			_a    (						self :							str  )->      Tuple:
     self.assertEqual(TensorType('tf'  )    ,       TensorType.TENSORFLOW  )
     self.assertEqual(TensorType('pt'  )    ,       TensorType.PYTORCH  )
     self.assertEqual(TensorType('np'  )    ,       TensorType.NUMPY  )






   @require_tokenizers
   def 			_a    (						self :							Optional[int]  )->      Optional[int]:
     import numpy as np

     _lowercase:			List[str]      =     BertTokenizer.from_pretrained('bert-base-cased'  )
     _lowercase:			Any      =     BertTokenizerFast.from_pretrained('bert-base-cased'  )

     # Python no tensor
     with self.subTest('BatchEncoding (Python, return_tensors=None)'  ):
       self.assert_dump_and_restore(tokenizer_p('Small example to encode'  )  )

     with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'  ):
       self.assert_dump_and_restore(
           tokenizer_p('Small example to encode'    ,       return_tensors=TensorType.NUMPY  )    ,       np.array_equal  )

     with self.subTest('BatchEncoding (Rust, return_tensors=None)'  ):
       self.assert_dump_and_restore(tokenizer_r('Small example to encode'  )  )

     with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'  ):
       self.assert_dump_and_restore(
           tokenizer_r('Small example to encode'    ,       return_tensors=TensorType.NUMPY  )    ,       np.array_equal  )






   @require_tf
   @require_tokenizers
   def 			_a    (						self :							Union[str, Any]  )->      Any:
     import tensorflow as tf

     def tf_array_equals(UpperCamelCase_ :							Optional[int]    ,       UpperCamelCase_ :							List[str]  ):
       return tf.reduce_all(tf.equal(UpperCamelCase_    ,       UpperCamelCase_  )  )

     _lowercase:			Union[str, Any]      =     BertTokenizer.from_pretrained('bert-base-cased'  )
     _lowercase:			Tuple      =     BertTokenizerFast.from_pretrained('bert-base-cased'  )

     with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'  ):
       self.assert_dump_and_restore(
           tokenizer_p('Small example to encode'    ,       return_tensors=TensorType.TENSORFLOW  )    ,       UpperCamelCase_  )

     with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'  ):
       self.assert_dump_and_restore(
           tokenizer_r('Small example to encode'    ,       return_tensors=TensorType.TENSORFLOW  )    ,       UpperCamelCase_  )






   @require_torch
   @require_tokenizers
   def 			_a    (						self :							List[str]  )->      Optional[int]:
     import torch

     _lowercase:			List[str]      =     BertTokenizer.from_pretrained('bert-base-cased'  )
     _lowercase:			List[str]      =     BertTokenizerFast.from_pretrained('bert-base-cased'  )

     with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'  ):
       self.assert_dump_and_restore(
           tokenizer_p('Small example to encode'    ,       return_tensors=TensorType.PYTORCH  )    ,       torch.equal  )

     with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'  ):
       self.assert_dump_and_restore(
           tokenizer_r('Small example to encode'    ,       return_tensors=TensorType.PYTORCH  )    ,       torch.equal  )






   @require_tokenizers
   def 			_a    (						self :							List[Any]  )->      str:
     _lowercase:			Dict      =     BertTokenizer.from_pretrained('bert-base-cased'  )
     _lowercase:			Optional[int]      =     BertTokenizerFast.from_pretrained('bert-base-cased'  )

     with self.subTest('Python Tokenizer'  ):
       self.assertFalse(tokenizer_p('Small example to_encode'  ).is_fast  )

     with self.subTest('Rust Tokenizer'  ):
       self.assertTrue(tokenizer_r('Small example to_encode'  ).is_fast  )






   @require_tokenizers
   def 			_a    (						self :							Tuple  )->      Tuple:
     _lowercase:			str      =     BertTokenizerFast.from_pretrained('bert-base-cased'  )
     _lowercase:			Optional[int]      =     tokenizer_r(['Test', '\xad', 'test']    ,       is_split_into_words=UpperCamelCase_  )

     self.assertEqual(encoded.word_to_tokens(0  )    ,       TokenSpan(start=1    ,       end=2  )  )
     self.assertEqual(encoded.word_to_tokens(1  )    ,       UpperCamelCase_  )
     self.assertEqual(encoded.word_to_tokens(2  )    ,       TokenSpan(start=2    ,       end=3  )  )






   def 			_a    (						self :							Optional[int]  )->      List[str]:
     _lowercase:			int      =     BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}  )
     _lowercase:			Any      =     batch.convert_to_tensors(tensor_type='np'  )
     self.assertEqual(tensor_batch['inputs'].shape    ,       (2, 3)  )
     self.assertEqual(tensor_batch['labels'].shape    ,       (2,)  )
     # test converting the converted
     with CaptureStderr() as cs:
       _lowercase:			Optional[int]      =     batch.convert_to_tensors(tensor_type='np'  )
     self.assertFalse(len(cs.err  )    ,       msg=f'should have no warning, but got {cs.err}'  )

     _lowercase:			Dict      =     BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}  )
     _lowercase:			List[Any]      =     batch.convert_to_tensors(tensor_type='np'    ,       prepend_batch_axis=UpperCamelCase_  )
     self.assertEqual(tensor_batch['inputs'].shape    ,       (1, 3)  )
     self.assertEqual(tensor_batch['labels'].shape    ,       (1,)  )






   @require_torch
   def 			_a    (						self :							Dict  )->      List[str]:
     _lowercase:			Optional[int]      =     BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}  )
     _lowercase:			Any      =     batch.convert_to_tensors(tensor_type='pt'  )
     self.assertEqual(tensor_batch['inputs'].shape    ,       (2, 3)  )
     self.assertEqual(tensor_batch['labels'].shape    ,       (2,)  )
     # test converting the converted
     with CaptureStderr() as cs:
       _lowercase:			str      =     batch.convert_to_tensors(tensor_type='pt'  )
     self.assertFalse(len(cs.err  )    ,       msg=f'should have no warning, but got {cs.err}'  )

     _lowercase:			List[Any]      =     BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}  )
     _lowercase:			Optional[int]      =     batch.convert_to_tensors(tensor_type='pt'    ,       prepend_batch_axis=UpperCamelCase_  )
     self.assertEqual(tensor_batch['inputs'].shape    ,       (1, 3)  )
     self.assertEqual(tensor_batch['labels'].shape    ,       (1,)  )






   @require_tf
   def 			_a    (						self :							Dict  )->      Union[str, Any]:
     _lowercase:			Any      =     BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}  )
     _lowercase:			Union[str, Any]      =     batch.convert_to_tensors(tensor_type='tf'  )
     self.assertEqual(tensor_batch['inputs'].shape    ,       (2, 3)  )
     self.assertEqual(tensor_batch['labels'].shape    ,       (2,)  )
     # test converting the converted
     with CaptureStderr() as cs:
       _lowercase:			int      =     batch.convert_to_tensors(tensor_type='tf'  )
     self.assertFalse(len(cs.err  )    ,       msg=f'should have no warning, but got {cs.err}'  )

     _lowercase:			Optional[Any]      =     BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}  )
     _lowercase:			List[str]      =     batch.convert_to_tensors(tensor_type='tf'    ,       prepend_batch_axis=UpperCamelCase_  )
     self.assertEqual(tensor_batch['inputs'].shape    ,       (1, 3)  )
     self.assertEqual(tensor_batch['labels'].shape    ,       (1,)  )






   @require_flax
   def 			_a    (						self :							List[Any]  )->      Optional[Any]:
     _lowercase:			int      =     BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}  )
     _lowercase:			int      =     batch.convert_to_tensors(tensor_type='jax'  )
     self.assertEqual(tensor_batch['inputs'].shape    ,       (2, 3)  )
     self.assertEqual(tensor_batch['labels'].shape    ,       (2,)  )
     # test converting the converted
     with CaptureStderr() as cs:
       _lowercase:			Any      =     batch.convert_to_tensors(tensor_type='jax'  )
     self.assertFalse(len(cs.err  )    ,       msg=f'should have no warning, but got {cs.err}'  )

     _lowercase:			Optional[Any]      =     BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}  )
     _lowercase:			Dict      =     batch.convert_to_tensors(tensor_type='jax'    ,       prepend_batch_axis=UpperCamelCase_  )
     self.assertEqual(tensor_batch['inputs'].shape    ,       (1, 3)  )
     self.assertEqual(tensor_batch['labels'].shape    ,       (1,)  )






   def 			_a    (						self :							List[Any]  )->      Any:
     _lowercase:			List[str]      =     [{'input_ids': np.array([0, 1, 2]  )}, {'input_ids': np.array([0, 1, 2, 3]  )}]
     _lowercase:			List[str]      =     BertTokenizer.from_pretrained('bert-base-cased'  )

     _lowercase:			List[Any]      =     tokenizer.pad(UpperCamelCase_    ,       padding=UpperCamelCase_  )
     self.assertTrue(isinstance(batch['input_ids']    ,       np.ndarray  )  )
     self.assertEqual(batch['input_ids'].tolist()    ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
     _lowercase:			Union[str, Any]      =     tokenizer.pad(UpperCamelCase_    ,       padding=UpperCamelCase_    ,       return_tensors='np'  )
     self.assertTrue(isinstance(batch['input_ids']    ,       np.ndarray  )  )
     self.assertEqual(batch['input_ids'].tolist()    ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )






   @require_torch
   def 			_a    (						self :							str  )->      List[str]:
     import torch

     _lowercase:			Dict      =     [{'input_ids': torch.tensor([0, 1, 2]  )}, {'input_ids': torch.tensor([0, 1, 2, 3]  )}]
     _lowercase:			Any      =     BertTokenizer.from_pretrained('bert-base-cased'  )

     _lowercase:			Optional[Any]      =     tokenizer.pad(UpperCamelCase_    ,       padding=UpperCamelCase_  )
     self.assertTrue(isinstance(batch['input_ids']    ,       torch.Tensor  )  )
     self.assertEqual(batch['input_ids'].tolist()    ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
     _lowercase:			Dict      =     tokenizer.pad(UpperCamelCase_    ,       padding=UpperCamelCase_    ,       return_tensors='pt'  )
     self.assertTrue(isinstance(batch['input_ids']    ,       torch.Tensor  )  )
     self.assertEqual(batch['input_ids'].tolist()    ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )






   @require_tf
   def 			_a    (						self :							Tuple  )->      List[str]:
     import tensorflow as tf

     _lowercase:			Dict      =     [{'input_ids': tf.constant([0, 1, 2]  )}, {'input_ids': tf.constant([0, 1, 2, 3]  )}]
     _lowercase:			Tuple      =     BertTokenizer.from_pretrained('bert-base-cased'  )

     _lowercase:			int      =     tokenizer.pad(UpperCamelCase_    ,       padding=UpperCamelCase_  )
     self.assertTrue(isinstance(batch['input_ids']    ,       tf.Tensor  )  )
     self.assertEqual(batch['input_ids'].numpy().tolist()    ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
     _lowercase:			Dict      =     tokenizer.pad(UpperCamelCase_    ,       padding=UpperCamelCase_    ,       return_tensors='tf'  )
     self.assertTrue(isinstance(batch['input_ids']    ,       tf.Tensor  )  )
     self.assertEqual(batch['input_ids'].numpy().tolist()    ,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )






   @require_tokenizers
   def 			_a    (						self :							List[str]  )->      Union[str, Any]:
     _lowercase:			Optional[Any]      =     Tokenizer(WordPiece(unk_token='[UNK]'  )  )
     PreTrainedTokenizerFast(tokenizer_object=UpperCamelCase_  )






   @require_tokenizers
   def 			_a    (						self :							str  )->      List[Any]:
     _lowercase:			Optional[Any]      =     Tokenizer(WordPiece(unk_token='[UNK]'  )  )
     with tempfile.TemporaryDirectory() as tmpdirname:
       bert_tokenizer.save(os.path.join(UpperCamelCase_    ,       'tokenizer.json'  )  )
       PreTrainedTokenizerFast(tokenizer_file=os.path.join(UpperCamelCase_    ,       'tokenizer.json'  )  )






def 		_lowerCamelCase					(       UpperCamelCase       ,  UpperCamelCase				)      ->		int:





  """simple docstring"""





  return (input_a, input_a).count(0				) % 2


def 		_lowerCamelCase					(       )      ->		None:





  """simple docstring"""





  assert xor_gate(0       ,  0				) == 0
  assert xor_gate(0       ,  1				) == 1
  assert xor_gate(1       ,  0				) == 1
  assert xor_gate(1       ,  1				) == 0


if __name__ == "__main__":
   print(xor_gate(0, 0))
   print(xor_gate(0, 1))
