





"""simple docstring"""




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 					_a		( _lowerCAmelCase      ,							unittest.TestCase							):
				UpperCamelCase								=       PreTrainedTokenizerFast
				UpperCamelCase								=       False
				UpperCamelCase								=       True
				UpperCamelCase								=       '''tokenizer_file'''






				def 							snake_case      (					self						:       Dict				)      -> Dict:




										'''simple docstring'''



										_UpperCamelCase     :   Union[str, Any]      =       False  # because we don't have pretrained_vocab_files_map
										super().setUp()
										_UpperCamelCase     :   Tuple      =       True

										_UpperCamelCase     :   Optional[int]      =       ['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
										_UpperCamelCase     :   Tuple      =       '''SaulLu/dummy-tokenizer-bytelevel-bpe'''

										# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
										_UpperCamelCase     :   Optional[int]      =       [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

										_UpperCamelCase     :   Union[str, Any]      =       PreTrainedTokenizerFast.from_pretrained(model_paths[0]				)
										tokenizer.save_pretrained(self.tmpdirname				)






				def 							snake_case      (					self						:       Tuple				)      -> str:




										'''simple docstring'''



										pass






				def 							snake_case      (					self						:       Dict				)      -> Tuple:




										'''simple docstring'''



										pass






				def 							snake_case      (					self						:       str				)      -> Optional[int]:




										'''simple docstring'''



										pass






				def 							snake_case      (					self						:       str				)      -> int:




										'''simple docstring'''



										pass






				def 							snake_case      (					self						:       List[Any]				)      -> Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase     :   List[str]      =       self.tmpdirname
										# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																						try:
																												_UpperCamelCase     :   List[Any]      =       tempfile.mkdtemp()
																												_UpperCamelCase     :   Optional[int]      =       self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__,					**lowerCAmelCase__				)

																												tokenizer.save_pretrained(self.tmpdirname				)
																												super().test_training_new_tokenizer()
																						finally:
																												# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																												# is restored
																												shutil.rmtree(self.tmpdirname				)
																												_UpperCamelCase     :   Union[str, Any]      =       tmpdirname_orig






				def 							snake_case      (					self						:       Tuple				)      -> Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase     :   Union[str, Any]      =       self.tmpdirname
										# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																						try:
																												_UpperCamelCase     :   Tuple      =       tempfile.mkdtemp()
																												_UpperCamelCase     :   Tuple      =       self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__,					**lowerCAmelCase__				)

																												tokenizer.save_pretrained(self.tmpdirname				)
																												super().test_training_new_tokenizer_with_special_tokens_change()
																						finally:
																												# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																												# is restored
																												shutil.rmtree(self.tmpdirname				)
																												_UpperCamelCase     :   Optional[int]      =       tmpdirname_orig






				def 							snake_case      (					self						:       Dict				)      -> Any:




										'''simple docstring'''



										_UpperCamelCase     :   Optional[Any]      =       self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name				)

										_UpperCamelCase     :   Any      =       ('''a''' for _ in range(1_0_0_0				))
										_UpperCamelCase     :   str      =       tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase__,					length=1_0_0_0,					vocab_size=5_0				)

										_UpperCamelCase     :   Optional[Any]      =       new_tokenizer.encode('''a🤗'''				)
										self.assertEqual(lowerCAmelCase__,					[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]				)




@require_tokenizers
class 					_a		( unittest.TestCase							):






				def 							snake_case      (					self						:       str				)      -> List[str]:




										'''simple docstring'''



										_UpperCamelCase     :   str      =       AutoTokenizer.from_pretrained('''bert-base-cased'''				)
										_UpperCamelCase     :   Optional[int]      =       json.loads(tokenizer._tokenizer.to_str()				)
										_UpperCamelCase     :   str      =       len(lowerCAmelCase__				)

										with tempfile.TemporaryDirectory() as tmp_dir:
																# Hack to save this in the tokenizer_config.json
																_UpperCamelCase     :   Tuple      =       ['''tokenizer.4.0.0.json''']
																tokenizer.save_pretrained(lowerCAmelCase__				)
																json.dump(lowerCAmelCase__,					open(os.path.join(lowerCAmelCase__,					'''tokenizer.4.0.0.json'''				),					'''w'''				)				)

																# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
																_UpperCamelCase     :   Optional[int]      =       AutoTokenizer.from_pretrained(lowerCAmelCase__				)
																self.assertEqual(len(lowerCAmelCase__				),					len(lowerCAmelCase__				) + 1				)
																_UpperCamelCase     :   Optional[int]      =       json.loads(new_tokenizer._tokenizer.to_str()				)
																self.assertIn('''huggingface''',					json_tokenizer['''model''']['''vocab''']				)

																# Will need to be adjusted if we reach v42 and this test is still here.
																# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
																shutil.move(os.path.join(lowerCAmelCase__,					'''tokenizer.4.0.0.json'''				),					os.path.join(lowerCAmelCase__,					'''tokenizer.42.0.0.json'''				)				)
																_UpperCamelCase     :   Any      =       ['''tokenizer.42.0.0.json''']
																tokenizer.save_pretrained(lowerCAmelCase__				)
																_UpperCamelCase     :   str      =       AutoTokenizer.from_pretrained(lowerCAmelCase__				)
																self.assertEqual(len(lowerCAmelCase__				),					len(lowerCAmelCase__				)				)
																_UpperCamelCase     :   Union[str, Any]      =       json.loads(new_tokenizer._tokenizer.to_str()				)
																self.assertNotIn('''huggingface''',					json_tokenizer['''model''']['''vocab''']				)






				def 							snake_case      (					self						:       Union[str, Any]				)      -> List[str]:




										'''simple docstring'''



										_UpperCamelCase     :   List[str]      =       '''hf-internal-testing/test-two-tokenizers'''

										# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
										_UpperCamelCase     :   List[str]      =       AutoTokenizer.from_pretrained(lowerCAmelCase__				)
										self.assertEqual(len(lowerCAmelCase__				),					2_8_9_9_7				)
										_UpperCamelCase     :   Union[str, Any]      =       json.loads(tokenizer._tokenizer.to_str()				)
										self.assertIn('''huggingface''',					json_tokenizer['''model''']['''vocab''']				)

										# Testing an older version by monkey-patching the version in the module it's used.
										import transformers as old_transformers

										_UpperCamelCase     :   str      =       '''3.0.0'''
										_UpperCamelCase     :   Dict      =       old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase__				)
										self.assertEqual(len(lowerCAmelCase__				),					2_8_9_9_6				)
										_UpperCamelCase     :   Dict      =       json.loads(old_tokenizer._tokenizer.to_str()				)
										self.assertNotIn('''huggingface''',					json_tokenizer['''model''']['''vocab''']				)




@require_tokenizers
class 					_a		( unittest.TestCase							):






				def 							snake_case      (					self						:       Optional[Any]				)      -> int:




										'''simple docstring'''



										_UpperCamelCase     :   int      =       PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''				)
										_UpperCamelCase     :   Optional[int]      =       '''The Matrix is a 1999 science fiction action film.'''

										with concurrent.futures.ThreadPoolExecutor() as executor:
																_UpperCamelCase     :   str      =       [executor.submit(self.fetch,					lowerCAmelCase__,					lowerCAmelCase__				) for i in range(1_0				)]
																_UpperCamelCase     :   Optional[Any]      =       [future.result() for future in futures]
																self.assertEqual(lowerCAmelCase__,					[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0				)]				)






				def 							snake_case      (					self						:       str,					lowerCAmelCase__						:       List[str],					lowerCAmelCase__						:       Optional[Any]				)      -> Tuple:




										'''simple docstring'''



										return tokenizer.encode(lowerCAmelCase__,					truncation='''longest_first''',					padding='''longest'''				)










"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase_	 ={
    """configuration_camembert""": ["""CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CamembertConfig""", """CamembertOnnxConfig"""],
}

try:
						if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase_	 =["""CamembertTokenizer"""]

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase_	 =["""CamembertTokenizerFast"""]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase_	 =[
						    """CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """CamembertForCausalLM""",
						    """CamembertForMaskedLM""",
						    """CamembertForMultipleChoice""",
						    """CamembertForQuestionAnswering""",
						    """CamembertForSequenceClassification""",
						    """CamembertForTokenClassification""",
						    """CamembertModel""",
						    """CamembertPreTrainedModel""",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase_	 =[
						    """TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """TFCamembertForCausalLM""",
						    """TFCamembertForMaskedLM""",
						    """TFCamembertForMultipleChoice""",
						    """TFCamembertForQuestionAnswering""",
						    """TFCamembertForSequenceClassification""",
						    """TFCamembertForTokenClassification""",
						    """TFCamembertModel""",
						    """TFCamembertPreTrainedModel""",
						]


if TYPE_CHECKING:
						from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig, CamembertOnnxConfig

						try:
												if not is_sentencepiece_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_camembert import CamembertTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_camembert_fast import CamembertTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_camembert import (
												    CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    CamembertForCausalLM,
												    CamembertForMaskedLM,
												    CamembertForMultipleChoice,
												    CamembertForQuestionAnswering,
												    CamembertForSequenceClassification,
												    CamembertForTokenClassification,
												    CamembertModel,
												    CamembertPreTrainedModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_camembert import (
												    TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFCamembertForCausalLM,
												    TFCamembertForMaskedLM,
												    TFCamembertForMultipleChoice,
												    TFCamembertForQuestionAnswering,
												    TFCamembertForSequenceClassification,
												    TFCamembertForTokenClassification,
												    TFCamembertModel,
												    TFCamembertPreTrainedModel,
												)

else:
						import sys

						UpperCamelCase_	 =_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



