import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 					snake_case__       (				_UpperCamelCase					,						unittest.TestCase							):
   _SCREAMING_SNAKE_CASE	: List[str]						    =      PreTrainedTokenizerFast
   _SCREAMING_SNAKE_CASE	: int						    =      False
   _SCREAMING_SNAKE_CASE	: List[Any]						    =      True
   _SCREAMING_SNAKE_CASE	: Any						    =      "tokenizer_file"



   def 					UpperCAmelCase__       (				self   :   int   )							->     List[Any]:




         '''simple docstring'''




         snake_case_     :			Tuple       =       False  # because we don't have pretrained_vocab_files_map
         super().setUp()
         snake_case_     :			Optional[int]       =       True

         snake_case_     :			Optional[int]       =       ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
         snake_case_     :			Dict       =       "SaulLu/dummy-tokenizer-bytelevel-bpe"

         # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
         snake_case_     :			Dict       =       [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

         snake_case_     :			Optional[int]       =       PreTrainedTokenizerFast.from_pretrained(model_paths[0]   )
         tokenizer.save_pretrained(self.tmpdirname   )



   def 					UpperCAmelCase__       (				self   :   Any   )							->     str:




         '''simple docstring'''




         pass



   def 					UpperCAmelCase__       (				self   :   Optional[Any]   )							->     int:




         '''simple docstring'''




         pass



   def 					UpperCAmelCase__       (				self   :   int   )							->     Optional[Any]:




         '''simple docstring'''




         pass



   def 					UpperCAmelCase__       (				self   :   Union[str, Any]   )							->     Any:




         '''simple docstring'''




         pass



   def 					UpperCAmelCase__       (				self   :   Dict   )							->     List[Any]:




         '''simple docstring'''




         snake_case_     :			List[Any]       =       self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
                     try:
                           snake_case_     :			str       =       tempfile.mkdtemp()
                           snake_case_     :			Union[str, Any]       =       self.rust_tokenizer_class.from_pretrained(A__			,	**A__   )

                           tokenizer.save_pretrained(self.tmpdirname   )
                           super().test_training_new_tokenizer()
                     finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname   )
                           snake_case_     :			Any       =       tmpdirname_orig



   def 					UpperCAmelCase__       (				self   :   int   )							->     Optional[int]:




         '''simple docstring'''




         snake_case_     :			Any       =       self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
                     try:
                           snake_case_     :			int       =       tempfile.mkdtemp()
                           snake_case_     :			Tuple       =       self.rust_tokenizer_class.from_pretrained(A__			,	**A__   )

                           tokenizer.save_pretrained(self.tmpdirname   )
                           super().test_training_new_tokenizer_with_special_tokens_change()
                     finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname   )
                           snake_case_     :			str       =       tmpdirname_orig



   def 					UpperCAmelCase__       (				self   :   str   )							->     Union[str, Any]:




         '''simple docstring'''




         snake_case_     :			List[str]       =       self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name   )

         snake_case_     :			List[Any]       =       ("a" for _ in range(10_00   ))
         snake_case_     :			Union[str, Any]       =       tokenizer.train_new_from_iterator(text_iterator=A__			,	length=10_00			,	vocab_size=50   )

         snake_case_     :			Any       =       new_tokenizer.encode("a🤗"   )
         self.assertEqual(A__			,	[64, 1_72, 2_53, 97, 2_45]   )



@require_tokenizers
class 					snake_case__       (				unittest.TestCase							):



   def 					UpperCAmelCase__       (				self   :   List[Any]   )							->     str:




         '''simple docstring'''




         snake_case_     :			Dict       =       AutoTokenizer.from_pretrained("bert-base-cased"   )
         snake_case_     :			Any       =       json.loads(tokenizer._tokenizer.to_str()   )
         snake_case_     :			Optional[int]       =       len(A__   )

         with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               snake_case_     :			Optional[Any]       =       ["tokenizer.4.0.0.json"]
               tokenizer.save_pretrained(A__   )
               json.dump(A__			,	open(os.path.join(A__			,	"tokenizer.4.0.0.json"   )			,	"w"   )   )

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               snake_case_     :			List[str]       =       AutoTokenizer.from_pretrained(A__   )
               self.assertEqual(len(A__   )			,	len(A__   ) + 1   )
               snake_case_     :			Any       =       json.loads(new_tokenizer._tokenizer.to_str()   )
               self.assertIn("huggingface"			,	json_tokenizer["model"]["vocab"]   )

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(A__			,	"tokenizer.4.0.0.json"   )			,	os.path.join(A__			,	"tokenizer.42.0.0.json"   )   )
               snake_case_     :			Any       =       ["tokenizer.42.0.0.json"]
               tokenizer.save_pretrained(A__   )
               snake_case_     :			Optional[Any]       =       AutoTokenizer.from_pretrained(A__   )
               self.assertEqual(len(A__   )			,	len(A__   )   )
               snake_case_     :			Any       =       json.loads(new_tokenizer._tokenizer.to_str()   )
               self.assertNotIn("huggingface"			,	json_tokenizer["model"]["vocab"]   )



   def 					UpperCAmelCase__       (				self   :   Any   )							->     Optional[int]:




         '''simple docstring'''




         snake_case_     :			List[Any]       =       "hf-internal-testing/test-two-tokenizers"

         # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
         snake_case_     :			List[Any]       =       AutoTokenizer.from_pretrained(A__   )
         self.assertEqual(len(A__   )			,	2_89_97   )
         snake_case_     :			Optional[Any]       =       json.loads(tokenizer._tokenizer.to_str()   )
         self.assertIn("huggingface"			,	json_tokenizer["model"]["vocab"]   )

         # Testing an older version by monkey-patching the version in the module it's used.
         import transformers as old_transformers

         snake_case_     :			List[Any]       =       "3.0.0"
         snake_case_     :			Tuple       =       old_transformers.models.auto.AutoTokenizer.from_pretrained(A__   )
         self.assertEqual(len(A__   )			,	2_89_96   )
         snake_case_     :			List[Any]       =       json.loads(old_tokenizer._tokenizer.to_str()   )
         self.assertNotIn("huggingface"			,	json_tokenizer["model"]["vocab"]   )



@require_tokenizers
class 					snake_case__       (				unittest.TestCase							):



   def 					UpperCAmelCase__       (				self   :   Any   )							->     Optional[int]:




         '''simple docstring'''




         snake_case_     :			Optional[int]       =       PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"   )
         snake_case_     :			Tuple       =       "The Matrix is a 1999 science fiction action film."

         with concurrent.futures.ThreadPoolExecutor() as executor:
               snake_case_     :			List[str]       =       [executor.submit(self.fetch			,	A__			,	A__   ) for i in range(10   )]
               snake_case_     :			Any       =       [future.result() for future in futures]
               self.assertEqual(A__			,	[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10   )]   )



   def 					UpperCAmelCase__       (				self   :   Any			,	A__   :   Any			,	A__   :   int   )							->     Optional[Any]:




         '''simple docstring'''




         return tokenizer.encode(A__			,	truncation="longest_first"			,	padding="longest"   )




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


UpperCAmelCase							=       logging.get_logger(__name__)

UpperCAmelCase							=       {
    "uclanlp/plbart-base": "https://huggingface.co/uclanlp/plbart-base/resolve/main/config.json",
    # See all PLBART models at https://huggingface.co/models?filter=plbart
}



class 					snake_case__       (				_UpperCamelCase							):
   _SCREAMING_SNAKE_CASE	: List[str]						    =      "plbart"
   _SCREAMING_SNAKE_CASE	: Union[str, Any]						    =      ["past_key_values"]
   _SCREAMING_SNAKE_CASE	: Dict						    =      {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}



   def __init__(				self   :   Dict			,	A__   :   Any=5_00_05			,	A__   :   Optional[int]=10_24			,	A__   :   Tuple=6			,	A__   :   Optional[Any]=30_72			,	A__   :   Dict=12			,	A__   :   Union[str, Any]=6			,	A__   :   Union[str, Any]=30_72			,	A__   :   Tuple=12			,	A__   :   List[str]=0.0			,	A__   :   Dict=0.0			,	A__   :   int=True			,	A__   :   Tuple=True			,	A__   :   List[str]="gelu"			,	A__   :   List[Any]=7_68			,	A__   :   Optional[Any]=0.1			,	A__   :   List[Any]=0.1			,	A__   :   Any=0.0			,	A__   :   int=0.02			,	A__   :   Optional[Any]=0.0			,	A__   :   List[str]=True			,	A__   :   Any=1			,	A__   :   Tuple=0			,	A__   :   Any=2			,	A__   :   int=2			,	**A__   :   Dict			,	)							->     int:




         '''simple docstring'''




         snake_case_     :			int       =       vocab_size
         snake_case_     :			List[Any]       =       max_position_embeddings
         snake_case_     :			str       =       d_model
         snake_case_     :			Tuple       =       encoder_ffn_dim
         snake_case_     :			Union[str, Any]       =       encoder_layers
         snake_case_     :			Dict       =       encoder_attention_heads
         snake_case_     :			Union[str, Any]       =       decoder_ffn_dim
         snake_case_     :			Union[str, Any]       =       decoder_layers
         snake_case_     :			List[str]       =       decoder_attention_heads
         snake_case_     :			List[Any]       =       dropout
         snake_case_     :			Union[str, Any]       =       attention_dropout
         snake_case_     :			Optional[int]       =       activation_dropout
         snake_case_     :			List[str]       =       activation_function
         snake_case_     :			Tuple       =       init_std
         snake_case_     :			str       =       encoder_layerdrop
         snake_case_     :			int       =       decoder_layerdrop
         snake_case_     :			Optional[int]       =       classifier_dropout
         snake_case_     :			List[str]       =       use_cache
         snake_case_     :			List[Any]       =       encoder_layers
         snake_case_     :			Tuple       =       scale_embedding  # scale factor will be sqrt(d_model) if True
         super().__init__(
             pad_token_id=A__			,	bos_token_id=A__			,	eos_token_id=A__			,	is_encoder_decoder=A__			,	forced_eos_token_id=A__			,	**A__			,	)



class 					snake_case__       (				_UpperCamelCase							):



   @property
   def 					UpperCAmelCase__       (				self   :   str   )							->     Mapping[str, Mapping[int, str]]:




         '''simple docstring'''




         return OrderedDict(
             [
                 ("input_ids", {0: "batch", 1: "sequence"}),
                 ("attention_mask", {0: "batch", 1: "sequence"}),
             ]   )



   @property
   def 					UpperCAmelCase__       (				self   :   List[str]   )							->     Mapping[str, Mapping[int, str]]:




         '''simple docstring'''




         if self.use_past:
               return OrderedDict(
                   [
                       ("last_hidden_state", {0: "batch", 1: "sequence"}),
                       ("past_keys", {0: "batch", 2: "sequence"}),
                       ("encoder_last_hidden_state", {0: "batch", 1: "sequence"}),
                   ]   )
         else:
               return OrderedDict(
                   [
                       ("last_hidden_state", {0: "batch", 1: "sequence"}),
                       ("encoder_last_hidden_state", {0: "batch", 1: "sequence"}),
                   ]   )



