





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
       from tokenizers import Tokenizer
       from tokenizers.models import WordPiece





class A_      (    unittest.TestCase     ):




   def _lowerCAmelCase   (self			:str					,     _UpperCamelCase			:Any				)->      Union[str, Any]:
         __A      =       list(tokenizer_class.max_model_input_sizes.keys()				)
         for model_name in sa_models[:1]:
               __A      =       tokenizer_class.from_pretrained(lowerCamelCase__				)
               self.assertIsNotNone(lowerCamelCase__				)
               self.assertIsInstance(lowerCamelCase__					,     lowerCamelCase__				)
               self.assertIsInstance(lowerCamelCase__					,     lowerCamelCase__				)

               for special_tok in tokenizer.all_special_tokens:
                     self.assertIsInstance(lowerCamelCase__					,     lowerCamelCase__				)
                     __A      =       tokenizer.convert_tokens_to_ids(lowerCamelCase__				)
                     self.assertIsInstance(lowerCamelCase__					,     lowerCamelCase__				)




   def _lowerCAmelCase   (self			:int					,     _UpperCamelCase			:Optional[int]					,     _UpperCamelCase			:Dict = None				)->      List[str]:
         __A      =       pickle.dumps(lowerCamelCase__				)
         self.assertIsNotNone(lowerCamelCase__				)

         __A      =       pickle.loads(lowerCamelCase__				)

         # Ensure is_fast is correctly restored
         self.assertEqual(be_restored.is_fast					,     be_original.is_fast				)

         # Ensure encodings are potentially correctly restored
         if be_original.is_fast:
               self.assertIsNotNone(be_restored.encodings				)
         else:
               self.assertIsNone(be_restored.encodings				)

         # Ensure the keys are the same
         for original_v, restored_v in zip(be_original.values()					,     be_restored.values()				):
               if equal_op:
                     self.assertTrue(equal_op(lowerCamelCase__					,     lowerCamelCase__				)				)
               else:
                     self.assertEqual(lowerCamelCase__					,     lowerCamelCase__				)




   @slow
   def _lowerCAmelCase   (self			:List[Any]				)->      Any:
         self.check_tokenizer_from_pretrained(lowerCamelCase__				)




   def _lowerCAmelCase   (self			:Any				)->      Tuple:
         self.assertEqual(TensorType('''tf'''				)					,     TensorType.TENSORFLOW				)
         self.assertEqual(TensorType('''pt'''				)					,     TensorType.PYTORCH				)
         self.assertEqual(TensorType('''np'''				)					,     TensorType.NUMPY				)




   @require_tokenizers
   def _lowerCAmelCase   (self			:Any				)->      List[str]:
         import numpy as np

         __A      =       BertTokenizer.from_pretrained('''bert-base-cased'''				)
         __A      =       BertTokenizerFast.from_pretrained('''bert-base-cased'''				)

         # Python no tensor
         with self.subTest('''BatchEncoding (Python, return_tensors=None)'''				):
               self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''				)				)

         with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''				):
               self.assert_dump_and_restore(
                   tokenizer_p('''Small example to encode'''					,     return_tensors=TensorType.NUMPY				)					,     np.array_equal				)

         with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''				):
               self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''				)				)

         with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''				):
               self.assert_dump_and_restore(
                   tokenizer_r('''Small example to encode'''					,     return_tensors=TensorType.NUMPY				)					,     np.array_equal				)




   @require_tf
   @require_tokenizers
   def _lowerCAmelCase   (self			:int				)->      str:
         import tensorflow as tf

         def tf_array_equals(_UpperCamelCase			:Tuple					,     _UpperCamelCase			:Optional[Any]				):
               return tf.reduce_all(tf.equal(lowerCamelCase__					,     lowerCamelCase__				)				)

         __A      =       BertTokenizer.from_pretrained('''bert-base-cased'''				)
         __A      =       BertTokenizerFast.from_pretrained('''bert-base-cased'''				)

         with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''				):
               self.assert_dump_and_restore(
                   tokenizer_p('''Small example to encode'''					,     return_tensors=TensorType.TENSORFLOW				)					,     lowerCamelCase__				)

         with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''				):
               self.assert_dump_and_restore(
                   tokenizer_r('''Small example to encode'''					,     return_tensors=TensorType.TENSORFLOW				)					,     lowerCamelCase__				)




   @require_torch
   @require_tokenizers
   def _lowerCAmelCase   (self			:List[Any]				)->      Dict:
         import torch

         __A      =       BertTokenizer.from_pretrained('''bert-base-cased'''				)
         __A      =       BertTokenizerFast.from_pretrained('''bert-base-cased'''				)

         with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''				):
               self.assert_dump_and_restore(
                   tokenizer_p('''Small example to encode'''					,     return_tensors=TensorType.PYTORCH				)					,     torch.equal				)

         with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''				):
               self.assert_dump_and_restore(
                   tokenizer_r('''Small example to encode'''					,     return_tensors=TensorType.PYTORCH				)					,     torch.equal				)




   @require_tokenizers
   def _lowerCAmelCase   (self			:Optional[Any]				)->      List[Any]:
         __A      =       BertTokenizer.from_pretrained('''bert-base-cased'''				)
         __A      =       BertTokenizerFast.from_pretrained('''bert-base-cased'''				)

         with self.subTest('''Python Tokenizer'''				):
               self.assertFalse(tokenizer_p('''Small example to_encode'''				).is_fast				)

         with self.subTest('''Rust Tokenizer'''				):
               self.assertTrue(tokenizer_r('''Small example to_encode'''				).is_fast				)




   @require_tokenizers
   def _lowerCAmelCase   (self			:str				)->      Optional[int]:
         __A      =       BertTokenizerFast.from_pretrained('''bert-base-cased'''				)
         __A      =       tokenizer_r(['''Test''', '''\xad''', '''test''']					,     is_split_into_words=lowerCamelCase__				)

         self.assertEqual(encoded.word_to_tokens(0				)					,     TokenSpan(start=1					,     end=2				)				)
         self.assertEqual(encoded.word_to_tokens(1				)					,     lowerCamelCase__				)
         self.assertEqual(encoded.word_to_tokens(2				)					,     TokenSpan(start=2					,     end=3				)				)




   def _lowerCAmelCase   (self			:Any				)->      str:
         __A      =       BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}				)
         __A      =       batch.convert_to_tensors(tensor_type='''np'''				)
         self.assertEqual(tensor_batch['''inputs'''].shape					,     (2, 3)				)
         self.assertEqual(tensor_batch['''labels'''].shape					,     (2,)				)
         # test converting the converted
         with CaptureStderr() as cs:
               __A      =       batch.convert_to_tensors(tensor_type='''np'''				)
         self.assertFalse(len(cs.err				)					,     msg=f"""should have no warning, but got {cs.err}"""				)

         __A      =       BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}				)
         __A      =       batch.convert_to_tensors(tensor_type='''np'''					,     prepend_batch_axis=lowerCamelCase__				)
         self.assertEqual(tensor_batch['''inputs'''].shape					,     (1, 3)				)
         self.assertEqual(tensor_batch['''labels'''].shape					,     (1,)				)




   @require_torch
   def _lowerCAmelCase   (self			:Optional[int]				)->      Optional[Any]:
         __A      =       BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}				)
         __A      =       batch.convert_to_tensors(tensor_type='''pt'''				)
         self.assertEqual(tensor_batch['''inputs'''].shape					,     (2, 3)				)
         self.assertEqual(tensor_batch['''labels'''].shape					,     (2,)				)
         # test converting the converted
         with CaptureStderr() as cs:
               __A      =       batch.convert_to_tensors(tensor_type='''pt'''				)
         self.assertFalse(len(cs.err				)					,     msg=f"""should have no warning, but got {cs.err}"""				)

         __A      =       BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}				)
         __A      =       batch.convert_to_tensors(tensor_type='''pt'''					,     prepend_batch_axis=lowerCamelCase__				)
         self.assertEqual(tensor_batch['''inputs'''].shape					,     (1, 3)				)
         self.assertEqual(tensor_batch['''labels'''].shape					,     (1,)				)




   @require_tf
   def _lowerCAmelCase   (self			:Dict				)->      str:
         __A      =       BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}				)
         __A      =       batch.convert_to_tensors(tensor_type='''tf'''				)
         self.assertEqual(tensor_batch['''inputs'''].shape					,     (2, 3)				)
         self.assertEqual(tensor_batch['''labels'''].shape					,     (2,)				)
         # test converting the converted
         with CaptureStderr() as cs:
               __A      =       batch.convert_to_tensors(tensor_type='''tf'''				)
         self.assertFalse(len(cs.err				)					,     msg=f"""should have no warning, but got {cs.err}"""				)

         __A      =       BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}				)
         __A      =       batch.convert_to_tensors(tensor_type='''tf'''					,     prepend_batch_axis=lowerCamelCase__				)
         self.assertEqual(tensor_batch['''inputs'''].shape					,     (1, 3)				)
         self.assertEqual(tensor_batch['''labels'''].shape					,     (1,)				)




   @require_flax
   def _lowerCAmelCase   (self			:int				)->      Tuple:
         __A      =       BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}				)
         __A      =       batch.convert_to_tensors(tensor_type='''jax'''				)
         self.assertEqual(tensor_batch['''inputs'''].shape					,     (2, 3)				)
         self.assertEqual(tensor_batch['''labels'''].shape					,     (2,)				)
         # test converting the converted
         with CaptureStderr() as cs:
               __A      =       batch.convert_to_tensors(tensor_type='''jax'''				)
         self.assertFalse(len(cs.err				)					,     msg=f"""should have no warning, but got {cs.err}"""				)

         __A      =       BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}				)
         __A      =       batch.convert_to_tensors(tensor_type='''jax'''					,     prepend_batch_axis=lowerCamelCase__				)
         self.assertEqual(tensor_batch['''inputs'''].shape					,     (1, 3)				)
         self.assertEqual(tensor_batch['''labels'''].shape					,     (1,)				)




   def _lowerCAmelCase   (self			:int				)->      Tuple:
         __A      =       [{"input_ids": np.array([0, 1, 2]				)}, {"input_ids": np.array([0, 1, 2, 3]				)}]
         __A      =       BertTokenizer.from_pretrained('''bert-base-cased'''				)

         __A      =       tokenizer.pad(lowerCamelCase__					,     padding=lowerCamelCase__				)
         self.assertTrue(isinstance(batch['''input_ids''']					,     np.ndarray				)				)
         self.assertEqual(batch['''input_ids'''].tolist()					,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)
         __A      =       tokenizer.pad(lowerCamelCase__					,     padding=lowerCamelCase__					,     return_tensors='''np'''				)
         self.assertTrue(isinstance(batch['''input_ids''']					,     np.ndarray				)				)
         self.assertEqual(batch['''input_ids'''].tolist()					,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)




   @require_torch
   def _lowerCAmelCase   (self			:Optional[Any]				)->      List[str]:
         import torch

         __A      =       [{"input_ids": torch.tensor([0, 1, 2]				)}, {"input_ids": torch.tensor([0, 1, 2, 3]				)}]
         __A      =       BertTokenizer.from_pretrained('''bert-base-cased'''				)

         __A      =       tokenizer.pad(lowerCamelCase__					,     padding=lowerCamelCase__				)
         self.assertTrue(isinstance(batch['''input_ids''']					,     torch.Tensor				)				)
         self.assertEqual(batch['''input_ids'''].tolist()					,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)
         __A      =       tokenizer.pad(lowerCamelCase__					,     padding=lowerCamelCase__					,     return_tensors='''pt'''				)
         self.assertTrue(isinstance(batch['''input_ids''']					,     torch.Tensor				)				)
         self.assertEqual(batch['''input_ids'''].tolist()					,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)




   @require_tf
   def _lowerCAmelCase   (self			:int				)->      Any:
         import tensorflow as tf

         __A      =       [{"input_ids": tf.constant([0, 1, 2]				)}, {"input_ids": tf.constant([0, 1, 2, 3]				)}]
         __A      =       BertTokenizer.from_pretrained('''bert-base-cased'''				)

         __A      =       tokenizer.pad(lowerCamelCase__					,     padding=lowerCamelCase__				)
         self.assertTrue(isinstance(batch['''input_ids''']					,     tf.Tensor				)				)
         self.assertEqual(batch['''input_ids'''].numpy().tolist()					,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)
         __A      =       tokenizer.pad(lowerCamelCase__					,     padding=lowerCamelCase__					,     return_tensors='''tf'''				)
         self.assertTrue(isinstance(batch['''input_ids''']					,     tf.Tensor				)				)
         self.assertEqual(batch['''input_ids'''].numpy().tolist()					,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)




   @require_tokenizers
   def _lowerCAmelCase   (self			:List[Any]				)->      int:
         __A      =       Tokenizer(WordPiece(unk_token='''[UNK]'''				)				)
         PreTrainedTokenizerFast(tokenizer_object=lowerCamelCase__				)






   @require_tokenizers
   def _lowerCAmelCase   (self			:List[Any]				)->      List[Any]:
         __A      =       Tokenizer(WordPiece(unk_token='''[UNK]'''				)				)
         with tempfile.TemporaryDirectory() as tmpdirname:
               bert_tokenizer.save(os.path.join(lowerCamelCase__					,     '''tokenizer.json'''				)				)
               PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowerCamelCase__					,     '''tokenizer.json'''				)				)












from unittest.mock import patch

import datasets
from datasets import Dataset



def _a       (			)			->		Optional[Any]:







      '''simple docstring'''






      __A      =       Dataset.from_dict({'''col_1''': [3, 2, 0, 1]}				)

      with patch('''tqdm.auto.tqdm'''				) as mock_tqdm:
            datasets.disable_progress_bar()
            dset.map(lambda lowerCamelCase				: {"col_2": x["col_1"] + 1}				)
            mock_tqdm.assert_not_called()

            mock_tqdm.reset_mock()

            datasets.enable_progress_bar()
            dset.map(lambda lowerCamelCase				: {"col_2": x["col_1"] + 1}				)
            mock_tqdm.assert_called()





