


'''simple docstring'''





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
   from tokenizers import Tokenizer
   from tokenizers.models import WordPiece


class     __lowerCAmelCase     ( unittest.TestCase					):






  """simple docstring"""





  def 					lowerCAmelCase__  (     self       :       str    ,				_lowerCAmelCase       :       str  )   -> int:




     """simple docstring"""
     snake_case_					    =					list(tokenizer_class.max_model_input_sizes.keys()  )
     for model_name in sa_models[:1]:
        snake_case_					    =					tokenizer_class.from_pretrained(_lowerCAmelCase  )
        self.assertIsNotNone(_lowerCAmelCase  )
        self.assertIsInstance(_lowerCAmelCase    ,				_lowerCAmelCase  )
        self.assertIsInstance(_lowerCAmelCase    ,				_lowerCAmelCase  )

        for special_tok in tokenizer.all_special_tokens:
           self.assertIsInstance(_lowerCAmelCase    ,				_lowerCAmelCase  )
           snake_case_					    =					tokenizer.convert_tokens_to_ids(_lowerCAmelCase  )
           self.assertIsInstance(_lowerCAmelCase    ,				_lowerCAmelCase  )



  def 					lowerCAmelCase__  (     self       :       List[str]    ,				_lowerCAmelCase       :       BatchEncoding    ,				_lowerCAmelCase       :       Optional[Callable] = None  )   -> Optional[Any]:




     """simple docstring"""
     snake_case_					    =					pickle.dumps(_lowerCAmelCase  )
     self.assertIsNotNone(_lowerCAmelCase  )

     snake_case_					    =					pickle.loads(_lowerCAmelCase  )

     # Ensure is_fast is correctly restored
     self.assertEqual(be_restored.is_fast    ,				be_original.is_fast  )

     # Ensure encodings are potentially correctly restored
     if be_original.is_fast:
        self.assertIsNotNone(be_restored.encodings  )
     else:
        self.assertIsNone(be_restored.encodings  )

     # Ensure the keys are the same
     for original_v, restored_v in zip(be_original.values()    ,				be_restored.values()  ):
        if equal_op:
           self.assertTrue(equal_op(_lowerCAmelCase    ,				_lowerCAmelCase  )  )
        else:
           self.assertEqual(_lowerCAmelCase    ,				_lowerCAmelCase  )



  @slow
  def 					lowerCAmelCase__  (     self       :       Any  )   -> Union[str, Any]:




     """simple docstring"""
     self.check_tokenizer_from_pretrained(_lowerCAmelCase  )



  def 					lowerCAmelCase__  (     self       :       Any  )   -> Tuple:




     """simple docstring"""
     self.assertEqual(TensorType("tf"  )    ,				TensorType.TENSORFLOW  )
     self.assertEqual(TensorType("pt"  )    ,				TensorType.PYTORCH  )
     self.assertEqual(TensorType("np"  )    ,				TensorType.NUMPY  )



  @require_tokenizers
  def 					lowerCAmelCase__  (     self       :       int  )   -> Dict:




     """simple docstring"""
     import numpy as np

     snake_case_					    =					BertTokenizer.from_pretrained("bert-base-cased"  )
     snake_case_					    =					BertTokenizerFast.from_pretrained("bert-base-cased"  )

     # Python no tensor
     with self.subTest("BatchEncoding (Python, return_tensors=None)"  ):
        self.assert_dump_and_restore(tokenizer_p("Small example to encode"  )  )

     with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"  ):
        self.assert_dump_and_restore(
            tokenizer_p("Small example to encode"    ,				return_tensors=TensorType.NUMPY  )    ,				np.array_equal  )

     with self.subTest("BatchEncoding (Rust, return_tensors=None)"  ):
        self.assert_dump_and_restore(tokenizer_r("Small example to encode"  )  )

     with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"  ):
        self.assert_dump_and_restore(
            tokenizer_r("Small example to encode"    ,				return_tensors=TensorType.NUMPY  )    ,				np.array_equal  )



  @require_tf
  @require_tokenizers
  def 					lowerCAmelCase__  (     self       :       str  )   -> Optional[Any]:




     """simple docstring"""
     import tensorflow as tf

     def tf_array_equals(_lowerCAmelCase       :       List[str]    ,				_lowerCAmelCase       :       Optional[int]  ):
        return tf.reduce_all(tf.equal(_lowerCAmelCase    ,				_lowerCAmelCase  )  )

     snake_case_					    =					BertTokenizer.from_pretrained("bert-base-cased"  )
     snake_case_					    =					BertTokenizerFast.from_pretrained("bert-base-cased"  )

     with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"  ):
        self.assert_dump_and_restore(
            tokenizer_p("Small example to encode"    ,				return_tensors=TensorType.TENSORFLOW  )    ,				_lowerCAmelCase  )

     with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"  ):
        self.assert_dump_and_restore(
            tokenizer_r("Small example to encode"    ,				return_tensors=TensorType.TENSORFLOW  )    ,				_lowerCAmelCase  )



  @require_torch
  @require_tokenizers
  def 					lowerCAmelCase__  (     self       :       str  )   -> str:




     """simple docstring"""
     import torch

     snake_case_					    =					BertTokenizer.from_pretrained("bert-base-cased"  )
     snake_case_					    =					BertTokenizerFast.from_pretrained("bert-base-cased"  )

     with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"  ):
        self.assert_dump_and_restore(
            tokenizer_p("Small example to encode"    ,				return_tensors=TensorType.PYTORCH  )    ,				torch.equal  )

     with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"  ):
        self.assert_dump_and_restore(
            tokenizer_r("Small example to encode"    ,				return_tensors=TensorType.PYTORCH  )    ,				torch.equal  )



  @require_tokenizers
  def 					lowerCAmelCase__  (     self       :       Dict  )   -> Dict:




     """simple docstring"""
     snake_case_					    =					BertTokenizer.from_pretrained("bert-base-cased"  )
     snake_case_					    =					BertTokenizerFast.from_pretrained("bert-base-cased"  )

     with self.subTest("Python Tokenizer"  ):
        self.assertFalse(tokenizer_p("Small example to_encode"  ).is_fast  )

     with self.subTest("Rust Tokenizer"  ):
        self.assertTrue(tokenizer_r("Small example to_encode"  ).is_fast  )



  @require_tokenizers
  def 					lowerCAmelCase__  (     self       :       str  )   -> Optional[Any]:




     """simple docstring"""
     snake_case_					    =					BertTokenizerFast.from_pretrained("bert-base-cased"  )
     snake_case_					    =					tokenizer_r(["Test", "\xad", "test"]    ,				is_split_into_words=_lowerCAmelCase  )

     self.assertEqual(encoded.word_to_tokens(0  )    ,				TokenSpan(start=1    ,				end=2  )  )
     self.assertEqual(encoded.word_to_tokens(1  )    ,				_lowerCAmelCase  )
     self.assertEqual(encoded.word_to_tokens(2  )    ,				TokenSpan(start=2    ,				end=3  )  )



  def 					lowerCAmelCase__  (     self       :       Dict  )   -> Dict:




     """simple docstring"""
     snake_case_					    =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}  )
     snake_case_					    =					batch.convert_to_tensors(tensor_type="np"  )
     self.assertEqual(tensor_batch["inputs"].shape    ,				(2, 3)  )
     self.assertEqual(tensor_batch["labels"].shape    ,				(2,)  )
     # test converting the converted
     with CaptureStderr() as cs:
        snake_case_					    =					batch.convert_to_tensors(tensor_type="np"  )
     self.assertFalse(len(cs.err  )    ,				msg=F'''should have no warning, but got {cs.err}'''  )

     snake_case_					    =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}  )
     snake_case_					    =					batch.convert_to_tensors(tensor_type="np"    ,				prepend_batch_axis=_lowerCAmelCase  )
     self.assertEqual(tensor_batch["inputs"].shape    ,				(1, 3)  )
     self.assertEqual(tensor_batch["labels"].shape    ,				(1,)  )



  @require_torch
  def 					lowerCAmelCase__  (     self       :       List[str]  )   -> int:




     """simple docstring"""
     snake_case_					    =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}  )
     snake_case_					    =					batch.convert_to_tensors(tensor_type="pt"  )
     self.assertEqual(tensor_batch["inputs"].shape    ,				(2, 3)  )
     self.assertEqual(tensor_batch["labels"].shape    ,				(2,)  )
     # test converting the converted
     with CaptureStderr() as cs:
        snake_case_					    =					batch.convert_to_tensors(tensor_type="pt"  )
     self.assertFalse(len(cs.err  )    ,				msg=F'''should have no warning, but got {cs.err}'''  )

     snake_case_					    =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}  )
     snake_case_					    =					batch.convert_to_tensors(tensor_type="pt"    ,				prepend_batch_axis=_lowerCAmelCase  )
     self.assertEqual(tensor_batch["inputs"].shape    ,				(1, 3)  )
     self.assertEqual(tensor_batch["labels"].shape    ,				(1,)  )



  @require_tf
  def 					lowerCAmelCase__  (     self       :       Optional[Any]  )   -> Tuple:




     """simple docstring"""
     snake_case_					    =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}  )
     snake_case_					    =					batch.convert_to_tensors(tensor_type="tf"  )
     self.assertEqual(tensor_batch["inputs"].shape    ,				(2, 3)  )
     self.assertEqual(tensor_batch["labels"].shape    ,				(2,)  )
     # test converting the converted
     with CaptureStderr() as cs:
        snake_case_					    =					batch.convert_to_tensors(tensor_type="tf"  )
     self.assertFalse(len(cs.err  )    ,				msg=F'''should have no warning, but got {cs.err}'''  )

     snake_case_					    =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}  )
     snake_case_					    =					batch.convert_to_tensors(tensor_type="tf"    ,				prepend_batch_axis=_lowerCAmelCase  )
     self.assertEqual(tensor_batch["inputs"].shape    ,				(1, 3)  )
     self.assertEqual(tensor_batch["labels"].shape    ,				(1,)  )



  @require_flax
  def 					lowerCAmelCase__  (     self       :       Tuple  )   -> Optional[int]:




     """simple docstring"""
     snake_case_					    =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}  )
     snake_case_					    =					batch.convert_to_tensors(tensor_type="jax"  )
     self.assertEqual(tensor_batch["inputs"].shape    ,				(2, 3)  )
     self.assertEqual(tensor_batch["labels"].shape    ,				(2,)  )
     # test converting the converted
     with CaptureStderr() as cs:
        snake_case_					    =					batch.convert_to_tensors(tensor_type="jax"  )
     self.assertFalse(len(cs.err  )    ,				msg=F'''should have no warning, but got {cs.err}'''  )

     snake_case_					    =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}  )
     snake_case_					    =					batch.convert_to_tensors(tensor_type="jax"    ,				prepend_batch_axis=_lowerCAmelCase  )
     self.assertEqual(tensor_batch["inputs"].shape    ,				(1, 3)  )
     self.assertEqual(tensor_batch["labels"].shape    ,				(1,)  )



  def 					lowerCAmelCase__  (     self       :       List[Any]  )   -> Tuple:




     """simple docstring"""
     snake_case_					    =					[{"input_ids": np.array([0, 1, 2]  )}, {"input_ids": np.array([0, 1, 2, 3]  )}]
     snake_case_					    =					BertTokenizer.from_pretrained("bert-base-cased"  )

     snake_case_					    =					tokenizer.pad(_lowerCAmelCase    ,				padding=_lowerCAmelCase  )
     self.assertTrue(isinstance(batch["input_ids"]    ,				np.ndarray  )  )
     self.assertEqual(batch["input_ids"].tolist()    ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
     snake_case_					    =					tokenizer.pad(_lowerCAmelCase    ,				padding=_lowerCAmelCase    ,				return_tensors="np"  )
     self.assertTrue(isinstance(batch["input_ids"]    ,				np.ndarray  )  )
     self.assertEqual(batch["input_ids"].tolist()    ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )



  @require_torch
  def 					lowerCAmelCase__  (     self       :       Union[str, Any]  )   -> Optional[int]:




     """simple docstring"""
     import torch

     snake_case_					    =					[{"input_ids": torch.tensor([0, 1, 2]  )}, {"input_ids": torch.tensor([0, 1, 2, 3]  )}]
     snake_case_					    =					BertTokenizer.from_pretrained("bert-base-cased"  )

     snake_case_					    =					tokenizer.pad(_lowerCAmelCase    ,				padding=_lowerCAmelCase  )
     self.assertTrue(isinstance(batch["input_ids"]    ,				torch.Tensor  )  )
     self.assertEqual(batch["input_ids"].tolist()    ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
     snake_case_					    =					tokenizer.pad(_lowerCAmelCase    ,				padding=_lowerCAmelCase    ,				return_tensors="pt"  )
     self.assertTrue(isinstance(batch["input_ids"]    ,				torch.Tensor  )  )
     self.assertEqual(batch["input_ids"].tolist()    ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )



  @require_tf
  def 					lowerCAmelCase__  (     self       :       str  )   -> List[Any]:




     """simple docstring"""
     import tensorflow as tf

     snake_case_					    =					[{"input_ids": tf.constant([0, 1, 2]  )}, {"input_ids": tf.constant([0, 1, 2, 3]  )}]
     snake_case_					    =					BertTokenizer.from_pretrained("bert-base-cased"  )

     snake_case_					    =					tokenizer.pad(_lowerCAmelCase    ,				padding=_lowerCAmelCase  )
     self.assertTrue(isinstance(batch["input_ids"]    ,				tf.Tensor  )  )
     self.assertEqual(batch["input_ids"].numpy().tolist()    ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
     snake_case_					    =					tokenizer.pad(_lowerCAmelCase    ,				padding=_lowerCAmelCase    ,				return_tensors="tf"  )
     self.assertTrue(isinstance(batch["input_ids"]    ,				tf.Tensor  )  )
     self.assertEqual(batch["input_ids"].numpy().tolist()    ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )



  @require_tokenizers
  def 					lowerCAmelCase__  (     self       :       Any  )   -> Union[str, Any]:




     """simple docstring"""
     snake_case_					    =					Tokenizer(WordPiece(unk_token="[UNK]"  )  )
     PreTrainedTokenizerFast(tokenizer_object=_lowerCAmelCase  )



  @require_tokenizers
  def 					lowerCAmelCase__  (     self       :       Optional[Any]  )   -> Any:




     """simple docstring"""
     snake_case_					    =					Tokenizer(WordPiece(unk_token="[UNK]"  )  )
     with tempfile.TemporaryDirectory() as tmpdirname:
        bert_tokenizer.save(os.path.join(_lowerCAmelCase    ,				"tokenizer.json"  )  )
        PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowerCAmelCase    ,				"tokenizer.json"  )  )








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


SCREAMING_SNAKE_CASE	:List[str]  		=     {
    '''configuration_prophetnet''': ['''PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ProphetNetConfig'''],
    '''tokenization_prophetnet''': ['''ProphetNetTokenizer'''],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   SCREAMING_SNAKE_CASE	:List[str]  		=     [
       '''PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''ProphetNetDecoder''',
       '''ProphetNetEncoder''',
       '''ProphetNetForCausalLM''',
       '''ProphetNetForConditionalGeneration''',
       '''ProphetNetModel''',
       '''ProphetNetPreTrainedModel''',
   ]


if TYPE_CHECKING:
   from .configuration_prophetnet import PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ProphetNetConfig
   from .tokenization_prophetnet import ProphetNetTokenizer

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_prophetnet import (
          PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
          ProphetNetDecoder,
          ProphetNetEncoder,
          ProphetNetForCausalLM,
          ProphetNetForConditionalGeneration,
          ProphetNetModel,
          ProphetNetPreTrainedModel,
      )

else:
   import sys

   SCREAMING_SNAKE_CASE	:int  		=     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




