

import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
     from tokenizers import Tokenizer
     from tokenizers.models import WordPiece



class 					lowercase		( unittest.TestCase      ):



 '''simple docstring'''


 def 						UpperCamelCase__							(self , __a				)				->      Union[str, Any]:






  """simple docstring"""





  UpperCAmelCase__           =       list(tokenizer_class.max_model_input_sizes.keys()				)
  for model_name in sa_models[:1]:
   UpperCAmelCase__           =       tokenizer_class.from_pretrained(__a				)
   self.assertIsNotNone(__a				)
   self.assertIsInstance(__a , __a				)
   self.assertIsInstance(__a , __a				)

   for special_tok in tokenizer.all_special_tokens:
    self.assertIsInstance(__a , __a				)
    UpperCAmelCase__           =       tokenizer.convert_tokens_to_ids(__a				)
    self.assertIsInstance(__a , __a				)




 def 						UpperCamelCase__							(self , __a , __a = None				)				->      Dict:






  """simple docstring"""





  UpperCAmelCase__           =       pickle.dumps(__a				)
  self.assertIsNotNone(__a				)

  UpperCAmelCase__           =       pickle.loads(__a				)

  # Ensure is_fast is correctly restored
  self.assertEqual(be_restored.is_fast , be_original.is_fast				)

  # Ensure encodings are potentially correctly restored
  if be_original.is_fast:
   self.assertIsNotNone(be_restored.encodings				)
  else:
   self.assertIsNone(be_restored.encodings				)

  # Ensure the keys are the same
  for original_v, restored_v in zip(be_original.values() , be_restored.values()				):
   if equal_op:
    self.assertTrue(equal_op(__a , __a				)				)
   else:
    self.assertEqual(__a , __a				)




 @slow
 def 						UpperCamelCase__							(self				)				->      Optional[int]:






  """simple docstring"""





  self.check_tokenizer_from_pretrained(__a				)




 def 						UpperCamelCase__							(self				)				->      str:






  """simple docstring"""





  self.assertEqual(TensorType('tf'				) , TensorType.TENSORFLOW				)
  self.assertEqual(TensorType('pt'				) , TensorType.PYTORCH				)
  self.assertEqual(TensorType('np'				) , TensorType.NUMPY				)




 @require_tokenizers
 def 						UpperCamelCase__							(self				)				->      List[str]:






  """simple docstring"""





  import numpy as np

  UpperCAmelCase__           =       BertTokenizer.from_pretrained('bert-base-cased'				)
  UpperCAmelCase__           =       BertTokenizerFast.from_pretrained('bert-base-cased'				)

  # Python no tensor
  with self.subTest('BatchEncoding (Python, return_tensors=None)'				):
   self.assert_dump_and_restore(tokenizer_p('Small example to encode'				)				)

  with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'				):
   self.assert_dump_and_restore(
       tokenizer_p('Small example to encode' , return_tensors=TensorType.NUMPY				) , np.array_equal				)

  with self.subTest('BatchEncoding (Rust, return_tensors=None)'				):
   self.assert_dump_and_restore(tokenizer_r('Small example to encode'				)				)

  with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'				):
   self.assert_dump_and_restore(
       tokenizer_r('Small example to encode' , return_tensors=TensorType.NUMPY				) , np.array_equal				)




 @require_tf
 @require_tokenizers
 def 						UpperCamelCase__							(self				)				->      Tuple:






  """simple docstring"""





  import tensorflow as tf

  def tf_array_equals(__a , __a				):
   return tf.reduce_all(tf.equal(__a , __a				)				)

  UpperCAmelCase__           =       BertTokenizer.from_pretrained('bert-base-cased'				)
  UpperCAmelCase__           =       BertTokenizerFast.from_pretrained('bert-base-cased'				)

  with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'				):
   self.assert_dump_and_restore(
       tokenizer_p('Small example to encode' , return_tensors=TensorType.TENSORFLOW				) , __a				)

  with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'				):
   self.assert_dump_and_restore(
       tokenizer_r('Small example to encode' , return_tensors=TensorType.TENSORFLOW				) , __a				)




 @require_torch
 @require_tokenizers
 def 						UpperCamelCase__							(self				)				->      Any:






  """simple docstring"""





  import torch

  UpperCAmelCase__           =       BertTokenizer.from_pretrained('bert-base-cased'				)
  UpperCAmelCase__           =       BertTokenizerFast.from_pretrained('bert-base-cased'				)

  with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'				):
   self.assert_dump_and_restore(
       tokenizer_p('Small example to encode' , return_tensors=TensorType.PYTORCH				) , torch.equal				)

  with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'				):
   self.assert_dump_and_restore(
       tokenizer_r('Small example to encode' , return_tensors=TensorType.PYTORCH				) , torch.equal				)




 @require_tokenizers
 def 						UpperCamelCase__							(self				)				->      Dict:






  """simple docstring"""





  UpperCAmelCase__           =       BertTokenizer.from_pretrained('bert-base-cased'				)
  UpperCAmelCase__           =       BertTokenizerFast.from_pretrained('bert-base-cased'				)

  with self.subTest('Python Tokenizer'				):
   self.assertFalse(tokenizer_p('Small example to_encode'				).is_fast				)

  with self.subTest('Rust Tokenizer'				):
   self.assertTrue(tokenizer_r('Small example to_encode'				).is_fast				)




 @require_tokenizers
 def 						UpperCamelCase__							(self				)				->      List[Any]:






  """simple docstring"""





  UpperCAmelCase__           =       BertTokenizerFast.from_pretrained('bert-base-cased'				)
  UpperCAmelCase__           =       tokenizer_r(['Test', '\xad', 'test'] , is_split_into_words=__a				)

  self.assertEqual(encoded.word_to_tokens(0				) , TokenSpan(start=1 , end=2				)				)
  self.assertEqual(encoded.word_to_tokens(1				) , __a				)
  self.assertEqual(encoded.word_to_tokens(2				) , TokenSpan(start=2 , end=3				)				)




 def 						UpperCamelCase__							(self				)				->      Dict:






  """simple docstring"""





  UpperCAmelCase__           =       BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}				)
  UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='np'				)
  self.assertEqual(tensor_batch['inputs'].shape , (2, 3)				)
  self.assertEqual(tensor_batch['labels'].shape , (2,)				)
  # test converting the converted
  with CaptureStderr() as cs:
   UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='np'				)
  self.assertFalse(len(cs.err				) , msg=F"should have no warning, but got {cs.err}"				)

  UpperCAmelCase__           =       BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}				)
  UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='np' , prepend_batch_axis=__a				)
  self.assertEqual(tensor_batch['inputs'].shape , (1, 3)				)
  self.assertEqual(tensor_batch['labels'].shape , (1,)				)




 @require_torch
 def 						UpperCamelCase__							(self				)				->      List[str]:






  """simple docstring"""





  UpperCAmelCase__           =       BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}				)
  UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='pt'				)
  self.assertEqual(tensor_batch['inputs'].shape , (2, 3)				)
  self.assertEqual(tensor_batch['labels'].shape , (2,)				)
  # test converting the converted
  with CaptureStderr() as cs:
   UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='pt'				)
  self.assertFalse(len(cs.err				) , msg=F"should have no warning, but got {cs.err}"				)

  UpperCAmelCase__           =       BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}				)
  UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='pt' , prepend_batch_axis=__a				)
  self.assertEqual(tensor_batch['inputs'].shape , (1, 3)				)
  self.assertEqual(tensor_batch['labels'].shape , (1,)				)




 @require_tf
 def 						UpperCamelCase__							(self				)				->      Union[str, Any]:






  """simple docstring"""





  UpperCAmelCase__           =       BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}				)
  UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='tf'				)
  self.assertEqual(tensor_batch['inputs'].shape , (2, 3)				)
  self.assertEqual(tensor_batch['labels'].shape , (2,)				)
  # test converting the converted
  with CaptureStderr() as cs:
   UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='tf'				)
  self.assertFalse(len(cs.err				) , msg=F"should have no warning, but got {cs.err}"				)

  UpperCAmelCase__           =       BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}				)
  UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='tf' , prepend_batch_axis=__a				)
  self.assertEqual(tensor_batch['inputs'].shape , (1, 3)				)
  self.assertEqual(tensor_batch['labels'].shape , (1,)				)




 @require_flax
 def 						UpperCamelCase__							(self				)				->      int:






  """simple docstring"""





  UpperCAmelCase__           =       BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}				)
  UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='jax'				)
  self.assertEqual(tensor_batch['inputs'].shape , (2, 3)				)
  self.assertEqual(tensor_batch['labels'].shape , (2,)				)
  # test converting the converted
  with CaptureStderr() as cs:
   UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='jax'				)
  self.assertFalse(len(cs.err				) , msg=F"should have no warning, but got {cs.err}"				)

  UpperCAmelCase__           =       BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}				)
  UpperCAmelCase__           =       batch.convert_to_tensors(tensor_type='jax' , prepend_batch_axis=__a				)
  self.assertEqual(tensor_batch['inputs'].shape , (1, 3)				)
  self.assertEqual(tensor_batch['labels'].shape , (1,)				)




 def 						UpperCamelCase__							(self				)				->      Any:






  """simple docstring"""





  UpperCAmelCase__           =       [{'input_ids': np.array([0, 1, 2]				)}, {'input_ids': np.array([0, 1, 2, 3]				)}]
  UpperCAmelCase__           =       BertTokenizer.from_pretrained('bert-base-cased'				)

  UpperCAmelCase__           =       tokenizer.pad(__a , padding=__a				)
  self.assertTrue(isinstance(batch['input_ids'] , np.ndarray				)				)
  self.assertEqual(batch['input_ids'].tolist() , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)
  UpperCAmelCase__           =       tokenizer.pad(__a , padding=__a , return_tensors='np'				)
  self.assertTrue(isinstance(batch['input_ids'] , np.ndarray				)				)
  self.assertEqual(batch['input_ids'].tolist() , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)




 @require_torch
 def 						UpperCamelCase__							(self				)				->      Any:






  """simple docstring"""





  import torch

  UpperCAmelCase__           =       [{'input_ids': torch.tensor([0, 1, 2]				)}, {'input_ids': torch.tensor([0, 1, 2, 3]				)}]
  UpperCAmelCase__           =       BertTokenizer.from_pretrained('bert-base-cased'				)

  UpperCAmelCase__           =       tokenizer.pad(__a , padding=__a				)
  self.assertTrue(isinstance(batch['input_ids'] , torch.Tensor				)				)
  self.assertEqual(batch['input_ids'].tolist() , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)
  UpperCAmelCase__           =       tokenizer.pad(__a , padding=__a , return_tensors='pt'				)
  self.assertTrue(isinstance(batch['input_ids'] , torch.Tensor				)				)
  self.assertEqual(batch['input_ids'].tolist() , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)




 @require_tf
 def 						UpperCamelCase__							(self				)				->      Any:






  """simple docstring"""





  import tensorflow as tf

  UpperCAmelCase__           =       [{'input_ids': tf.constant([0, 1, 2]				)}, {'input_ids': tf.constant([0, 1, 2, 3]				)}]
  UpperCAmelCase__           =       BertTokenizer.from_pretrained('bert-base-cased'				)

  UpperCAmelCase__           =       tokenizer.pad(__a , padding=__a				)
  self.assertTrue(isinstance(batch['input_ids'] , tf.Tensor				)				)
  self.assertEqual(batch['input_ids'].numpy().tolist() , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)
  UpperCAmelCase__           =       tokenizer.pad(__a , padding=__a , return_tensors='tf'				)
  self.assertTrue(isinstance(batch['input_ids'] , tf.Tensor				)				)
  self.assertEqual(batch['input_ids'].numpy().tolist() , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]				)




 @require_tokenizers
 def 						UpperCamelCase__							(self				)				->      Union[str, Any]:






  """simple docstring"""





  UpperCAmelCase__           =       Tokenizer(WordPiece(unk_token='[UNK]'				)				)
  PreTrainedTokenizerFast(tokenizer_object=__a				)




 @require_tokenizers
 def 						UpperCamelCase__							(self				)				->      List[Any]:






  """simple docstring"""





  UpperCAmelCase__           =       Tokenizer(WordPiece(unk_token='[UNK]'				)				)
  with tempfile.TemporaryDirectory() as tmpdirname:
   bert_tokenizer.save(os.path.join(__a , 'tokenizer.json'				)				)
   PreTrainedTokenizerFast(tokenizer_file=os.path.join(__a , 'tokenizer.json'				)				)






from collections import OrderedDict
from dataclasses import fields
from typing import Any, Tuple

import numpy as np

from .import_utils import is_torch_available
def 		UpperCamelCase_(    snake_case__:						Optional[int]     ) ->				Tuple:
 if is_torch_available():
  import torch

  if isinstance(snake_case__						,					torch.Tensor     ):
   return True

 return isinstance(snake_case__						,					np.ndarray     )







class 					lowercase		( _UpperCamelCase      ):



 '''simple docstring'''


 def 						UpperCamelCase__							(self				)				->      int:






  """simple docstring"""





  UpperCAmelCase__           =       fields(self				)

  # Safety and consistency checks
  if not len(__a				):
   raise ValueError(F"{self.__class__.__name__} has no fields."				)

  UpperCAmelCase__           =       getattr(self , class_fields[0].name				)
  UpperCAmelCase__           =       all(getattr(self , field.name				) is None for field in class_fields[1:]				)

  if other_fields_are_none and isinstance(__a , __a				):
   for key, value in first_field.items():
    UpperCAmelCase__           =       value
  else:
   for field in class_fields:
    UpperCAmelCase__           =       getattr(self , field.name				)
    if v is not None:
     UpperCAmelCase__           =       v




 def __delitem__(self , *__a , **__a				)				->      Optional[int]:






  """simple docstring"""





  raise Exception(F"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance."				)




 def 						UpperCamelCase__							(self , *__a , **__a				)				->      str:






  """simple docstring"""





  raise Exception(F"You cannot use ``setdefault`` on a {self.__class__.__name__} instance."				)




 def 						UpperCamelCase__							(self , *__a , **__a				)				->      Optional[int]:






  """simple docstring"""





  raise Exception(F"You cannot use ``pop`` on a {self.__class__.__name__} instance."				)




 def 						UpperCamelCase__							(self , *__a , **__a				)				->      Tuple:






  """simple docstring"""





  raise Exception(F"You cannot use ``update`` on a {self.__class__.__name__} instance."				)




 def __getitem__(self , __a				)				->      Tuple:






  """simple docstring"""





  if isinstance(__a , __a				):
   UpperCAmelCase__           =       dict(self.items()				)
   return inner_dict[k]
  else:
   return self.to_tuple()[k]




 def __setattr__(self , __a , __a				)				->      Any:






  """simple docstring"""





  if name in self.keys() and value is not None:
   # Don't call self.__setitem__ to avoid recursion errors
   super().__setitem__(__a , __a				)
  super().__setattr__(__a , __a				)




 def __setitem__(self , __a , __a				)				->      Union[str, Any]:






  """simple docstring"""





  super().__setitem__(__a , __a				)
  # Don't call self.__setattr__ to avoid recursion errors
  super().__setattr__(__a , __a				)




 def 						UpperCamelCase__							(self				)				->      Tuple[Any]:






  """simple docstring"""





  return tuple(self[k] for k in self.keys()				)



