
import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
      from tokenizers import Tokenizer
      from tokenizers.models import WordPiece





class 							A__	(			unittest.TestCase):
      def 		UpperCamelCase__							(   self					,						__magic_name__   ):
       lowerCamelCase	:			int           =   list(tokenizer_class.max_model_input_sizes.keys()   )
       for model_name in sa_models[:1]:
        lowerCamelCase	:			Optional[int]           =   tokenizer_class.from_pretrained(__magic_name__   )
        self.assertIsNotNone(__magic_name__   )
        self.assertIsInstance(__magic_name__					,						__magic_name__   )
        self.assertIsInstance(__magic_name__					,						__magic_name__   )

        for special_tok in tokenizer.all_special_tokens:
         self.assertIsInstance(__magic_name__					,						__magic_name__   )
         lowerCamelCase	:			int           =   tokenizer.convert_tokens_to_ids(__magic_name__   )
         self.assertIsInstance(__magic_name__					,						__magic_name__   )
      def 		UpperCamelCase__							(   self					,						__magic_name__					,						__magic_name__ = None   ):
       lowerCamelCase	:			Any           =   pickle.dumps(__magic_name__   )
       self.assertIsNotNone(__magic_name__   )

       lowerCamelCase	:			Dict           =   pickle.loads(__magic_name__   )

       # Ensure is_fast is correctly restored
       self.assertEqual(be_restored.is_fast					,						be_original.is_fast   )

       # Ensure encodings are potentially correctly restored
       if be_original.is_fast:
        self.assertIsNotNone(be_restored.encodings   )
       else:
        self.assertIsNone(be_restored.encodings   )

       # Ensure the keys are the same
       for original_v, restored_v in zip(be_original.values()					,						be_restored.values()   ):
        if equal_op:
         self.assertTrue(equal_op(__magic_name__					,						__magic_name__   )   )
        else:
         self.assertEqual(__magic_name__					,						__magic_name__   )
      @slow
      def 		UpperCamelCase__							(   self   ):
       self.check_tokenizer_from_pretrained(__magic_name__   )
      def 		UpperCamelCase__							(   self   ):
       self.assertEqual(TensorType("""tf"""   )					,						TensorType.TENSORFLOW   )
       self.assertEqual(TensorType("""pt"""   )					,						TensorType.PYTORCH   )
       self.assertEqual(TensorType("""np"""   )					,						TensorType.NUMPY   )
      @require_tokenizers
      def 		UpperCamelCase__							(   self   ):
       import numpy as np

       lowerCamelCase	:			Any           =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
       lowerCamelCase	:			List[str]           =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

       # Python no tensor
       with self.subTest("""BatchEncoding (Python, return_tensors=None)"""   ):
        self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""   )   )

       with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""   ):
        self.assert_dump_and_restore(
            tokenizer_p("""Small example to encode"""					,						return_tensors=TensorType.NUMPY   )					,						np.array_equal   )

       with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""   ):
        self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""   )   )

       with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""   ):
        self.assert_dump_and_restore(
            tokenizer_r("""Small example to encode"""					,						return_tensors=TensorType.NUMPY   )					,						np.array_equal   )
      @require_tf
      @require_tokenizers
      def 		UpperCamelCase__							(   self   ):
       import tensorflow as tf

       def tf_array_equals(__magic_name__					,						__magic_name__   ):
        return tf.reduce_all(tf.equal(__magic_name__					,						__magic_name__   )   )

       lowerCamelCase	:			Tuple           =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
       lowerCamelCase	:			List[Any]           =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

       with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""   ):
        self.assert_dump_and_restore(
            tokenizer_p("""Small example to encode"""					,						return_tensors=TensorType.TENSORFLOW   )					,						__magic_name__   )

       with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""   ):
        self.assert_dump_and_restore(
            tokenizer_r("""Small example to encode"""					,						return_tensors=TensorType.TENSORFLOW   )					,						__magic_name__   )
      @require_torch
      @require_tokenizers
      def 		UpperCamelCase__							(   self   ):
       import torch

       lowerCamelCase	:			str           =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
       lowerCamelCase	:			Dict           =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

       with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""   ):
        self.assert_dump_and_restore(
            tokenizer_p("""Small example to encode"""					,						return_tensors=TensorType.PYTORCH   )					,						torch.equal   )

       with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""   ):
        self.assert_dump_and_restore(
            tokenizer_r("""Small example to encode"""					,						return_tensors=TensorType.PYTORCH   )					,						torch.equal   )
      @require_tokenizers
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			Union[str, Any]           =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
       lowerCamelCase	:			Union[str, Any]           =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

       with self.subTest("""Python Tokenizer"""   ):
        self.assertFalse(tokenizer_p("""Small example to_encode"""   ).is_fast   )

       with self.subTest("""Rust Tokenizer"""   ):
        self.assertTrue(tokenizer_r("""Small example to_encode"""   ).is_fast   )
      @require_tokenizers
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			int           =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )
       lowerCamelCase	:			str           =   tokenizer_r(["""Test""", """\xad""", """test"""]					,						is_split_into_words=__magic_name__   )

       self.assertEqual(encoded.word_to_tokens(0   )					,						TokenSpan(start=1					,						end=2   )   )
       self.assertEqual(encoded.word_to_tokens(1   )					,						__magic_name__   )
       self.assertEqual(encoded.word_to_tokens(2   )					,						TokenSpan(start=2					,						end=3   )   )
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			Optional[int]           =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
       lowerCamelCase	:			List[str]           =   batch.convert_to_tensors(tensor_type="""np"""   )
       self.assertEqual(tensor_batch["""inputs"""].shape					,						(2, 3)   )
       self.assertEqual(tensor_batch["""labels"""].shape					,						(2,)   )
       # test converting the converted
       with CaptureStderr() as cs:
        lowerCamelCase	:			Optional[Any]           =   batch.convert_to_tensors(tensor_type="""np"""   )
       self.assertFalse(len(cs.err   )					,						msg=F'''should have no warning, but got {cs.err}'''   )

       lowerCamelCase	:			Union[str, Any]           =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
       lowerCamelCase	:			List[Any]           =   batch.convert_to_tensors(tensor_type="""np"""					,						prepend_batch_axis=__magic_name__   )
       self.assertEqual(tensor_batch["""inputs"""].shape					,						(1, 3)   )
       self.assertEqual(tensor_batch["""labels"""].shape					,						(1,)   )
      @require_torch
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			Any           =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
       lowerCamelCase	:			Union[str, Any]           =   batch.convert_to_tensors(tensor_type="""pt"""   )
       self.assertEqual(tensor_batch["""inputs"""].shape					,						(2, 3)   )
       self.assertEqual(tensor_batch["""labels"""].shape					,						(2,)   )
       # test converting the converted
       with CaptureStderr() as cs:
        lowerCamelCase	:			Any           =   batch.convert_to_tensors(tensor_type="""pt"""   )
       self.assertFalse(len(cs.err   )					,						msg=F'''should have no warning, but got {cs.err}'''   )

       lowerCamelCase	:			str           =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
       lowerCamelCase	:			int           =   batch.convert_to_tensors(tensor_type="""pt"""					,						prepend_batch_axis=__magic_name__   )
       self.assertEqual(tensor_batch["""inputs"""].shape					,						(1, 3)   )
       self.assertEqual(tensor_batch["""labels"""].shape					,						(1,)   )
      @require_tf
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			str           =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
       lowerCamelCase	:			int           =   batch.convert_to_tensors(tensor_type="""tf"""   )
       self.assertEqual(tensor_batch["""inputs"""].shape					,						(2, 3)   )
       self.assertEqual(tensor_batch["""labels"""].shape					,						(2,)   )
       # test converting the converted
       with CaptureStderr() as cs:
        lowerCamelCase	:			Tuple           =   batch.convert_to_tensors(tensor_type="""tf"""   )
       self.assertFalse(len(cs.err   )					,						msg=F'''should have no warning, but got {cs.err}'''   )

       lowerCamelCase	:			List[Any]           =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
       lowerCamelCase	:			List[Any]           =   batch.convert_to_tensors(tensor_type="""tf"""					,						prepend_batch_axis=__magic_name__   )
       self.assertEqual(tensor_batch["""inputs"""].shape					,						(1, 3)   )
       self.assertEqual(tensor_batch["""labels"""].shape					,						(1,)   )
      @require_flax
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			Union[str, Any]           =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
       lowerCamelCase	:			Optional[int]           =   batch.convert_to_tensors(tensor_type="""jax"""   )
       self.assertEqual(tensor_batch["""inputs"""].shape					,						(2, 3)   )
       self.assertEqual(tensor_batch["""labels"""].shape					,						(2,)   )
       # test converting the converted
       with CaptureStderr() as cs:
        lowerCamelCase	:			List[str]           =   batch.convert_to_tensors(tensor_type="""jax"""   )
       self.assertFalse(len(cs.err   )					,						msg=F'''should have no warning, but got {cs.err}'''   )

       lowerCamelCase	:			Dict           =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
       lowerCamelCase	:			Union[str, Any]           =   batch.convert_to_tensors(tensor_type="""jax"""					,						prepend_batch_axis=__magic_name__   )
       self.assertEqual(tensor_batch["""inputs"""].shape					,						(1, 3)   )
       self.assertEqual(tensor_batch["""labels"""].shape					,						(1,)   )
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			Optional[Any]           =   [{"""input_ids""": np.array([0, 1, 2]   )}, {"""input_ids""": np.array([0, 1, 2, 3]   )}]
       lowerCamelCase	:			Any           =   BertTokenizer.from_pretrained("""bert-base-cased"""   )

       lowerCamelCase	:			Optional[int]           =   tokenizer.pad(__magic_name__					,						padding=__magic_name__   )
       self.assertTrue(isinstance(batch["""input_ids"""]					,						np.ndarray   )   )
       self.assertEqual(batch["""input_ids"""].tolist()					,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
       lowerCamelCase	:			Optional[Any]           =   tokenizer.pad(__magic_name__					,						padding=__magic_name__					,						return_tensors="""np"""   )
       self.assertTrue(isinstance(batch["""input_ids"""]					,						np.ndarray   )   )
       self.assertEqual(batch["""input_ids"""].tolist()					,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
      @require_torch
      def 		UpperCamelCase__							(   self   ):
       import torch

       lowerCamelCase	:			str           =   [{"""input_ids""": torch.tensor([0, 1, 2]   )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]   )}]
       lowerCamelCase	:			List[Any]           =   BertTokenizer.from_pretrained("""bert-base-cased"""   )

       lowerCamelCase	:			Optional[Any]           =   tokenizer.pad(__magic_name__					,						padding=__magic_name__   )
       self.assertTrue(isinstance(batch["""input_ids"""]					,						torch.Tensor   )   )
       self.assertEqual(batch["""input_ids"""].tolist()					,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
       lowerCamelCase	:			List[str]           =   tokenizer.pad(__magic_name__					,						padding=__magic_name__					,						return_tensors="""pt"""   )
       self.assertTrue(isinstance(batch["""input_ids"""]					,						torch.Tensor   )   )
       self.assertEqual(batch["""input_ids"""].tolist()					,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
      @require_tf
      def 		UpperCamelCase__							(   self   ):
       import tensorflow as tf

       lowerCamelCase	:			int           =   [{"""input_ids""": tf.constant([0, 1, 2]   )}, {"""input_ids""": tf.constant([0, 1, 2, 3]   )}]
       lowerCamelCase	:			List[str]           =   BertTokenizer.from_pretrained("""bert-base-cased"""   )

       lowerCamelCase	:			List[Any]           =   tokenizer.pad(__magic_name__					,						padding=__magic_name__   )
       self.assertTrue(isinstance(batch["""input_ids"""]					,						tf.Tensor   )   )
       self.assertEqual(batch["""input_ids"""].numpy().tolist()					,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
       lowerCamelCase	:			Dict           =   tokenizer.pad(__magic_name__					,						padding=__magic_name__					,						return_tensors="""tf"""   )
       self.assertTrue(isinstance(batch["""input_ids"""]					,						tf.Tensor   )   )
       self.assertEqual(batch["""input_ids"""].numpy().tolist()					,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
      @require_tokenizers
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			Optional[int]           =   Tokenizer(WordPiece(unk_token="""[UNK]"""   )   )
       PreTrainedTokenizerFast(tokenizer_object=__magic_name__   )






      @require_tokenizers
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			Optional[int]           =   Tokenizer(WordPiece(unk_token="""[UNK]"""   )   )
       with tempfile.TemporaryDirectory() as tmpdirname:
        bert_tokenizer.save(os.path.join(__magic_name__					,						"""tokenizer.json"""   )   )
        PreTrainedTokenizerFast(tokenizer_file=os.path.join(__magic_name__					,						"""tokenizer.json"""   )   )








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowerCamelCase           ={
    """configuration_musicgen""": [
        """MUSICGEN_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """MusicgenConfig""",
        """MusicgenDecoderConfig""",
    ],
    """processing_musicgen""": ["""MusicgenProcessor"""],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _lowerCamelCase           =[
          """MUSICGEN_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """MusicgenForConditionalGeneration""",
          """MusicgenForCausalLM""",
          """MusicgenModel""",
          """MusicgenPreTrainedModel""",
      ]

if TYPE_CHECKING:
      from .configuration_musicgen import (
          MUSICGEN_PRETRAINED_CONFIG_ARCHIVE_MAP,
          MusicgenConfig,
          MusicgenDecoderConfig,
      )
      from .processing_musicgen import MusicgenProcessor

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_musicgen import (
                MUSICGEN_PRETRAINED_MODEL_ARCHIVE_LIST,
                MusicgenForCausalLM,
                MusicgenForConditionalGeneration,
                MusicgenModel,
                MusicgenPreTrainedModel,
            )

else:
      import sys

      _lowerCamelCase           =_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




