

import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class      _a				( snake_case_    ,							unittest.TestCase							):




      """simple docstring"""

      _lowerCamelCase					: str					=					PreTrainedTokenizerFast
      _lowerCamelCase					: int					=					False
      _lowerCamelCase					: Optional[Any]					=					True
      _lowerCamelCase					: Tuple					=					'tokenizer_file'





      def 		__A  (      self       :							List[str]  ):
            A_	     =							False  # because we don't have pretrained_vocab_files_map
            super().setUp()
            A_	     =							True

            A_	     =							["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
            A_	     =							"SaulLu/dummy-tokenizer-bytelevel-bpe"

            # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
            A_	     =							[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

            A_	     =							PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
            tokenizer.save_pretrained(self.tmpdirname  )





      def 		__A  (      self       :							Optional[Any]  ):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass





      def 		__A  (      self       :							Optional[int]  ):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass





      def 		__A  (      self       :							List[Any]  ):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass





      def 		__A  (      self       :							str  ):
            # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
            pass





      def 		__A  (      self       :							List[str]  ):
            A_	     =							self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                        try:
                              A_	     =							tempfile.mkdtemp()
                              A_	     =							self.rust_tokenizer_class.from_pretrained(UpperCAmelCase	, **UpperCAmelCase  )

                              tokenizer.save_pretrained(self.tmpdirname  )
                              super().test_training_new_tokenizer()
                        finally:
                              # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                              # is restored
                              shutil.rmtree(self.tmpdirname  )
                              A_	     =							tmpdirname_orig





      def 		__A  (      self       :							Union[str, Any]  ):
            A_	     =							self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                        try:
                              A_	     =							tempfile.mkdtemp()
                              A_	     =							self.rust_tokenizer_class.from_pretrained(UpperCAmelCase	, **UpperCAmelCase  )

                              tokenizer.save_pretrained(self.tmpdirname  )
                              super().test_training_new_tokenizer_with_special_tokens_change()
                        finally:
                              # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                              # is restored
                              shutil.rmtree(self.tmpdirname  )
                              A_	     =							tmpdirname_orig







      def 		__A  (      self       :							List[Any]  ):
            A_	     =							self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

            A_	     =							("a" for _ in range(1000  ))
            A_	     =							tokenizer.train_new_from_iterator(text_iterator=UpperCAmelCase	, length=1000	, vocab_size=50  )

            A_	     =							new_tokenizer.encode("a🤗"  )
            self.assertEqual(UpperCAmelCase	, [64, 172, 253, 97, 245]  )




@require_tokenizers
class      _a				( unittest.TestCase							):




      """simple docstring"""

      def 		__A  (      self       :							Any  ):
            A_	     =							AutoTokenizer.from_pretrained("bert-base-cased"  )
            A_	     =							json.loads(tokenizer._tokenizer.to_str()  )
            A_	     =							len(UpperCAmelCase  )

            with tempfile.TemporaryDirectory() as tmp_dir:
                  # Hack to save this in the tokenizer_config.json
                  A_	     =							["tokenizer.4.0.0.json"]
                  tokenizer.save_pretrained(UpperCAmelCase  )
                  json.dump(UpperCAmelCase	, open(os.path.join(UpperCAmelCase	, "tokenizer.4.0.0.json"  )	, "w"  )  )

                  # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                  A_	     =							AutoTokenizer.from_pretrained(UpperCAmelCase  )
                  self.assertEqual(len(UpperCAmelCase  )	, len(UpperCAmelCase  ) + 1  )
                  A_	     =							json.loads(new_tokenizer._tokenizer.to_str()  )
                  self.assertIn("huggingface"	, json_tokenizer["model"]["vocab"]  )

                  # Will need to be adjusted if we reach v42 and this test is still here.
                  # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                  shutil.move(os.path.join(UpperCAmelCase	, "tokenizer.4.0.0.json"  )	, os.path.join(UpperCAmelCase	, "tokenizer.42.0.0.json"  )  )
                  A_	     =							["tokenizer.42.0.0.json"]
                  tokenizer.save_pretrained(UpperCAmelCase  )
                  A_	     =							AutoTokenizer.from_pretrained(UpperCAmelCase  )
                  self.assertEqual(len(UpperCAmelCase  )	, len(UpperCAmelCase  )  )
                  A_	     =							json.loads(new_tokenizer._tokenizer.to_str()  )
                  self.assertNotIn("huggingface"	, json_tokenizer["model"]["vocab"]  )







      def 		__A  (      self       :							Any  ):
            # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
            A_	     =							"hf-internal-testing/test-two-tokenizers"

            # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
            A_	     =							AutoTokenizer.from_pretrained(UpperCAmelCase  )
            self.assertEqual(len(UpperCAmelCase  )	, 28997  )
            A_	     =							json.loads(tokenizer._tokenizer.to_str()  )
            self.assertIn("huggingface"	, json_tokenizer["model"]["vocab"]  )

            # Testing an older version by monkey-patching the version in the module it's used.
            import transformers as old_transformers

            A_	     =							"3.0.0"
            A_	     =							old_transformers.models.auto.AutoTokenizer.from_pretrained(UpperCAmelCase  )
            self.assertEqual(len(UpperCAmelCase  )	, 28996  )
            A_	     =							json.loads(old_tokenizer._tokenizer.to_str()  )
            self.assertNotIn("huggingface"	, json_tokenizer["model"]["vocab"]  )




@require_tokenizers
class      _a				( unittest.TestCase							):




      """simple docstring"""

      def 		__A  (      self       :							Union[str, Any]  ):
            # See https://github.com/huggingface/transformers/pull/12550
            # and https://github.com/huggingface/tokenizers/issues/537
            A_	     =							PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"  )
            A_	     =							"The Matrix is a 1999 science fiction action film."

            with concurrent.futures.ThreadPoolExecutor() as executor:
                  A_	     =							[executor.submit(self.fetch	, UpperCAmelCase	, UpperCAmelCase  ) for i in range(10  )]
                  A_	     =							[future.result() for future in futures]
                  self.assertEqual(UpperCAmelCase	, [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )







      def 		__A  (      self       :							str	, UpperCAmelCase       :							str	, UpperCAmelCase       :							Dict  ):
            return tokenizer.encode(UpperCAmelCase	, truncation="longest_first"	, padding="longest"  )


import warnings

from ...utils import logging
from .image_processing_mobilenet_va import MobileNetVaImageProcessor


__a						:Optional[int]            = logging.get_logger(__name__)

class      _a				( snake_case_							):




      """simple docstring"""

      def __init__(      self       :							List[Any]	, *UpperCAmelCase       :							Any	, **UpperCAmelCase       :							int  ):
            warnings.warn(
                "The class MobileNetV2FeatureExtractor is deprecated and will be removed in version 5 of Transformers."
                " Please use MobileNetV2ImageProcessor instead."	, UpperCAmelCase	, )
            super().__init__(*UpperCAmelCase	, **UpperCAmelCase  )