
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin






class 						UpperCAmelCase	(		A_				):
   A__      :	str		  =  ["image_processor", "tokenizer"]
   A__      :	Tuple		  =  "AutoImageProcessor"
   A__      :	Optional[Any]		  =  "AutoTokenizer"



   def __init__(self	:      List[Any]					,		snake_case__	:      Any=None					,		snake_case__	:      Union[str, Any]=None					,		**snake_case__	:      Tuple				)	->			Optional[int]:


          '''simple docstring'''


          snake_case :     int           =   None
          if "feature_extractor" in kwargs:
                 warnings.warn(
                     "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
                     " instead."					,		snake_case__					,		)
                 snake_case :     List[str]           =   kwargs.pop("feature_extractor"				)

          snake_case :     Union[str, Any]           =   image_processor if image_processor is not None else feature_extractor
          if image_processor is None:
                 raise ValueError("You need to specify an `image_processor`."				)
          if tokenizer is None:
                 raise ValueError("You need to specify a `tokenizer`."				)

          super().__init__(snake_case__					,		snake_case__				)
          snake_case :     str           =   self.image_processor
          snake_case :     Any           =   False



   def __call__(self	:      str					,		*snake_case__	:      str					,		**snake_case__	:      Optional[int]				)	->			Optional[Any]:


          '''simple docstring'''


          if self._in_target_context_manager:
                 return self.current_processor(*snake_case__					,		**snake_case__				)

          snake_case :     Dict           =   kwargs.pop("images"					,		snake_case__				)
          snake_case :     str           =   kwargs.pop("text"					,		snake_case__				)
          if len(snake_case__				) > 0:
                 snake_case :     List[str]           =   args[0]
                 snake_case :     Optional[Any]           =   args[1:]

          if images is None and text is None:
                 raise ValueError("You need to specify either an `images` or `text` input to process."				)

          if images is not None:
                 snake_case :     Tuple           =   self.image_processor(snake_case__					,		*snake_case__					,		**snake_case__				)
          if text is not None:
                 snake_case :     Optional[Any]           =   self.tokenizer(snake_case__					,		**snake_case__				)

          if text is None:
                 return inputs
          elif images is None:
                 return encodings
          else:
                 snake_case :     Tuple           =   encodings["input_ids"]
                 return inputs



   def        _SCREAMING_SNAKE_CASE     (self	:      int					,		*snake_case__	:      Tuple					,		**snake_case__	:      int				)	->			str:


          '''simple docstring'''


          return self.tokenizer.batch_decode(*snake_case__					,		**snake_case__				)



   def        _SCREAMING_SNAKE_CASE     (self	:      List[str]					,		*snake_case__	:      str					,		**snake_case__	:      List[Any]				)	->			List[str]:


          '''simple docstring'''


          return self.tokenizer.decode(*snake_case__					,		**snake_case__				)



   @contextmanager
   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			str:


          '''simple docstring'''


          warnings.warn(
              "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
              "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
              "your images inputs, or in a separate call."				)
          snake_case :     Optional[int]           =   True
          snake_case :     Tuple           =   self.tokenizer
          yield
          snake_case :     Optional[Any]           =   self.image_processor
          snake_case :     int           =   False



   @property
   def        _SCREAMING_SNAKE_CASE     (self	:      List[str]				)	->			Optional[int]:


          '''simple docstring'''


          warnings.warn(
              "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."					,		snake_case__					,		)
          return self.image_processor_class



   @property
   def        _SCREAMING_SNAKE_CASE     (self	:      Dict				)	->			Optional[int]:


          '''simple docstring'''


          warnings.warn(
              "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."					,		snake_case__					,		)
          return self.image_processor







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						UpperCAmelCase	(		A_ ,unittest.TestCase				):
   A__      :	Dict		  =  PreTrainedTokenizerFast
   A__      :	str		  =  False
   A__      :	List[Any]		  =  True
   A__      :	List[str]		  =  "tokenizer_file"



   def        _SCREAMING_SNAKE_CASE     (self	:      Dict				)	->			Union[str, Any]:


          '''simple docstring'''


          snake_case :     Union[str, Any]           =   False  # because we don't have pretrained_vocab_files_map
          super().setUp()
          snake_case :     Optional[int]           =   True

          snake_case :     List[str]           =   ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
          snake_case :     Optional[Any]           =   "SaulLu/dummy-tokenizer-bytelevel-bpe"

          # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
          snake_case :     int           =   [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

          snake_case :     List[Any]           =   PreTrainedTokenizerFast.from_pretrained(model_paths[0]				)
          tokenizer.save_pretrained(self.tmpdirname				)



   def        _SCREAMING_SNAKE_CASE     (self	:      Dict				)	->			int:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Any				)	->			int:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Any				)	->			Union[str, Any]:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Tuple				)	->			List[str]:


          '''simple docstring'''


          pass



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			List[str]:


          '''simple docstring'''


          snake_case :     Optional[int]           =   self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                        try:
                               snake_case :     Any           =   tempfile.mkdtemp()
                               snake_case :     int           =   self.rust_tokenizer_class.from_pretrained(snake_case__					,		**snake_case__				)

                               tokenizer.save_pretrained(self.tmpdirname				)
                               super().test_training_new_tokenizer()
                        finally:
                               # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                               # is restored
                               shutil.rmtree(self.tmpdirname				)
                               snake_case :     int           =   tmpdirname_orig



   def        _SCREAMING_SNAKE_CASE     (self	:      Tuple				)	->			Dict:


          '''simple docstring'''


          snake_case :     Optional[int]           =   self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                        try:
                               snake_case :     Union[str, Any]           =   tempfile.mkdtemp()
                               snake_case :     str           =   self.rust_tokenizer_class.from_pretrained(snake_case__					,		**snake_case__				)

                               tokenizer.save_pretrained(self.tmpdirname				)
                               super().test_training_new_tokenizer_with_special_tokens_change()
                        finally:
                               # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                               # is restored
                               shutil.rmtree(self.tmpdirname				)
                               snake_case :     Dict           =   tmpdirname_orig



   def        _SCREAMING_SNAKE_CASE     (self	:      int				)	->			Union[str, Any]:


          '''simple docstring'''


          snake_case :     int           =   self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name				)

          snake_case :     int           =   ("a" for _ in range(10_00				))
          snake_case :     Optional[int]           =   tokenizer.train_new_from_iterator(text_iterator=snake_case__					,		length=10_00					,		vocab_size=50				)

          snake_case :     str           =   new_tokenizer.encode("a🤗"				)
          self.assertEqual(snake_case__					,		[64, 1_72, 2_53, 97, 2_45]				)






@require_tokenizers
class 						UpperCAmelCase	(		unittest.TestCase				):



   def        _SCREAMING_SNAKE_CASE     (self	:      Union[str, Any]				)	->			List[str]:


          '''simple docstring'''


          snake_case :     List[str]           =   AutoTokenizer.from_pretrained("bert-base-cased"				)
          snake_case :     Union[str, Any]           =   json.loads(tokenizer._tokenizer.to_str()				)
          snake_case :     Tuple           =   len(snake_case__				)

          with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 snake_case :     List[str]           =   ["tokenizer.4.0.0.json"]
                 tokenizer.save_pretrained(snake_case__				)
                 json.dump(snake_case__					,		open(os.path.join(snake_case__					,		"tokenizer.4.0.0.json"				)					,		"w"				)				)

                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 snake_case :     Optional[Any]           =   AutoTokenizer.from_pretrained(snake_case__				)
                 self.assertEqual(len(snake_case__				)					,		len(snake_case__				) + 1				)
                 snake_case :     Union[str, Any]           =   json.loads(new_tokenizer._tokenizer.to_str()				)
                 self.assertIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)

                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(snake_case__					,		"tokenizer.4.0.0.json"				)					,		os.path.join(snake_case__					,		"tokenizer.42.0.0.json"				)				)
                 snake_case :     Optional[Any]           =   ["tokenizer.42.0.0.json"]
                 tokenizer.save_pretrained(snake_case__				)
                 snake_case :     Dict           =   AutoTokenizer.from_pretrained(snake_case__				)
                 self.assertEqual(len(snake_case__				)					,		len(snake_case__				)				)
                 snake_case :     Optional[int]           =   json.loads(new_tokenizer._tokenizer.to_str()				)
                 self.assertNotIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			List[str]:


          '''simple docstring'''


          snake_case :     int           =   "hf-internal-testing/test-two-tokenizers"

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          snake_case :     int           =   AutoTokenizer.from_pretrained(snake_case__				)
          self.assertEqual(len(snake_case__				)					,		2_89_97				)
          snake_case :     Optional[int]           =   json.loads(tokenizer._tokenizer.to_str()				)
          self.assertIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)

          # Testing an older version by monkey-patching the version in the module it's used.
          import transformers as old_transformers

          snake_case :     Dict           =   "3.0.0"
          snake_case :     Union[str, Any]           =   old_transformers.models.auto.AutoTokenizer.from_pretrained(snake_case__				)
          self.assertEqual(len(snake_case__				)					,		2_89_96				)
          snake_case :     Optional[Any]           =   json.loads(old_tokenizer._tokenizer.to_str()				)
          self.assertNotIn("huggingface"					,		json_tokenizer["model"]["vocab"]				)






@require_tokenizers
class 						UpperCAmelCase	(		unittest.TestCase				):



   def        _SCREAMING_SNAKE_CASE     (self	:      Optional[int]				)	->			List[Any]:


          '''simple docstring'''


          snake_case :     Tuple           =   PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"				)
          snake_case :     List[Any]           =   "The Matrix is a 1999 science fiction action film."

          with concurrent.futures.ThreadPoolExecutor() as executor:
                 snake_case :     Union[str, Any]           =   [executor.submit(self.fetch					,		snake_case__					,		snake_case__				) for i in range(10				)]
                 snake_case :     Optional[Any]           =   [future.result() for future in futures]
                 self.assertEqual(snake_case__					,		[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10				)]				)



   def        _SCREAMING_SNAKE_CASE     (self	:      str					,		snake_case__	:      List[str]					,		snake_case__	:      str				)	->			List[str]:


          '''simple docstring'''


          return tokenizer.encode(snake_case__					,		truncation="longest_first"					,		padding="longest"				)





