import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 				_SCREAMING_SNAKE_CASE( snake_case__							,   unittest.TestCase						):
   UpperCamelCase_					:    Dict			      = PreTrainedTokenizerFast
   UpperCamelCase_					:    Optional[int]			      = False
   UpperCamelCase_					:    Optional[int]			      = True
   UpperCamelCase_					:    Any			      = 'tokenizer_file'
   def _lowercase(     self     )						->Any:
        a	:     List[str]						      =    False  # because we don't have pretrained_vocab_files_map
        super().setUp()
        a	:     Optional[int]						      =    True

        a	:     Tuple						      =    ['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
        a	:     Optional[int]						      =    '''SaulLu/dummy-tokenizer-bytelevel-bpe'''

        # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
        a	:     Tuple						      =    [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

        a	:     Tuple						      =    PreTrainedTokenizerFast.from_pretrained(model_paths[0]     )
        tokenizer.save_pretrained(self.tmpdirname     )
   def _lowercase(     self     )						->Dict:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass
   def _lowercase(     self     )						->int:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass
   def _lowercase(     self     )						->List[str]:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass
   def _lowercase(     self     )						->Tuple:
        # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
        pass
   def _lowercase(     self     )						->Union[str, Any]:
        a	:     Optional[int]						      =    self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
                  try:
                       a	:     Dict						      =    tempfile.mkdtemp()
                       a	:     Any						      =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,  **lowerCAmelCase_     )

                       tokenizer.save_pretrained(self.tmpdirname     )
                       super().test_training_new_tokenizer()
                  finally:
                       # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                       # is restored
                       shutil.rmtree(self.tmpdirname     )
                       a	:     List[str]						      =    tmpdirname_orig
   def _lowercase(     self     )						->Tuple:
        a	:     int						      =    self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
                  try:
                       a	:     Optional[int]						      =    tempfile.mkdtemp()
                       a	:     Any						      =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,  **lowerCAmelCase_     )

                       tokenizer.save_pretrained(self.tmpdirname     )
                       super().test_training_new_tokenizer_with_special_tokens_change()
                  finally:
                       # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                       # is restored
                       shutil.rmtree(self.tmpdirname     )
                       a	:     Union[str, Any]						      =    tmpdirname_orig


   def _lowercase(     self     )						->Union[str, Any]:
        a	:     List[str]						      =    self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name     )

        a	:     Dict						      =    ('''a''' for _ in range(10_00     ))
        a	:     str						      =    tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase_				,  length=10_00				,  vocab_size=50     )

        a	:     Any						      =    new_tokenizer.encode('''a🤗'''     )
        self.assertEqual(lowerCAmelCase_				,  [64, 1_72, 2_53, 97, 2_45]     )






@require_tokenizers
class 				_SCREAMING_SNAKE_CASE( unittest.TestCase						):
   def _lowercase(     self     )						->Dict:
        a	:     List[Any]						      =    AutoTokenizer.from_pretrained('''bert-base-cased'''     )
        a	:     List[str]						      =    json.loads(tokenizer._tokenizer.to_str()     )
        a	:     List[str]						      =    len(lowerCAmelCase_     )

        with tempfile.TemporaryDirectory() as tmp_dir:
             # Hack to save this in the tokenizer_config.json
             a	:     str						      =    ['''tokenizer.4.0.0.json''']
             tokenizer.save_pretrained(lowerCAmelCase_     )
             json.dump(lowerCAmelCase_				,  open(os.path.join(lowerCAmelCase_				,  '''tokenizer.4.0.0.json'''     )				,  '''w'''     )     )

             # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
             a	:     Union[str, Any]						      =    AutoTokenizer.from_pretrained(lowerCAmelCase_     )
             self.assertEqual(len(lowerCAmelCase_     )				,  len(lowerCAmelCase_     ) + 1     )
             a	:     str						      =    json.loads(new_tokenizer._tokenizer.to_str()     )
             self.assertIn('''huggingface'''				,  json_tokenizer['''model''']['''vocab''']     )

             # Will need to be adjusted if we reach v42 and this test is still here.
             # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
             shutil.move(os.path.join(lowerCAmelCase_				,  '''tokenizer.4.0.0.json'''     )				,  os.path.join(lowerCAmelCase_				,  '''tokenizer.42.0.0.json'''     )     )
             a	:     List[str]						      =    ['''tokenizer.42.0.0.json''']
             tokenizer.save_pretrained(lowerCAmelCase_     )
             a	:     List[Any]						      =    AutoTokenizer.from_pretrained(lowerCAmelCase_     )
             self.assertEqual(len(lowerCAmelCase_     )				,  len(lowerCAmelCase_     )     )
             a	:     List[Any]						      =    json.loads(new_tokenizer._tokenizer.to_str()     )
             self.assertNotIn('''huggingface'''				,  json_tokenizer['''model''']['''vocab''']     )


   def _lowercase(     self     )						->Optional[int]:
        # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
        a	:     str						      =    '''hf-internal-testing/test-two-tokenizers'''

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        a	:     Dict						      =    AutoTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertEqual(len(lowerCAmelCase_     )				,  2_89_97     )
        a	:     Optional[Any]						      =    json.loads(tokenizer._tokenizer.to_str()     )
        self.assertIn('''huggingface'''				,  json_tokenizer['''model''']['''vocab''']     )

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        a	:     int						      =    '''3.0.0'''
        a	:     Tuple						      =    old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertEqual(len(lowerCAmelCase_     )				,  2_89_96     )
        a	:     List[str]						      =    json.loads(old_tokenizer._tokenizer.to_str()     )
        self.assertNotIn('''huggingface'''				,  json_tokenizer['''model''']['''vocab''']     )






@require_tokenizers
class 				_SCREAMING_SNAKE_CASE( unittest.TestCase						):
   def _lowercase(     self     )						->Any:
        # See https://github.com/huggingface/transformers/pull/12550
        # and https://github.com/huggingface/tokenizers/issues/537
        a	:     str						      =    PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''     )
        a	:     Union[str, Any]						      =    '''The Matrix is a 1999 science fiction action film.'''

        with concurrent.futures.ThreadPoolExecutor() as executor:
             a	:     Dict						      =    [executor.submit(self.fetch				,  lowerCAmelCase_				,  lowerCAmelCase_     ) for i in range(10     )]
             a	:     List[str]						      =    [future.result() for future in futures]
             self.assertEqual(lowerCAmelCase_				,  [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10     )]     )


   def _lowercase(     self				,  lowerCAmelCase_				,  lowerCAmelCase_     )						->List[Any]:
        return tokenizer.encode(lowerCAmelCase_				,  truncation='''longest_first'''				,  padding='''longest'''     )
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


UpperCAmelCase				 =   {
    """configuration_gptsan_japanese""": ["""GPTSAN_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP""", """GPTSanJapaneseConfig"""],
    """tokenization_gptsan_japanese""": ["""GPTSanJapaneseTokenizer"""],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase				 =   [
      """GPTSAN_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """GPTSanJapaneseForConditionalGeneration""",
      """GPTSanJapaneseModel""",
      """GPTSanJapanesePreTrainedModel""",
  ]
  UpperCAmelCase				 =   [
      """GPTSanJapaneseTokenizer""",
  ]


if TYPE_CHECKING:
  from .configuration_gptsan_japanese import GPTSAN_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTSanJapaneseConfig
  from .tokenization_gptsan_japanese import GPTSanJapaneseTokenizer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_gptsan_japanese import (
        GPTSAN_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST,
        GPTSanJapaneseForConditionalGeneration,
        GPTSanJapaneseModel,
        GPTSanJapanesePreTrainedModel,
    )
    from .tokenization_gptsan_japanese import GPTSanJapaneseTokenizer


else:
  import sys

  UpperCAmelCase				 =   _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)