
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 	A__				(	_lowercase		,  unittest.TestCase      ):
				lowercase            =    PreTrainedTokenizerFast
				lowercase            =    False
				lowercase            =    True
				lowercase            =    'tokenizer_file'






				def 				_lowerCamelCase						(		self   : int  ):






							'''simple docstring'''







							lowerCAmelCase__ : Union[str, Any]   =						False  # because we don't have pretrained_vocab_files_map
							super().setUp()
							lowerCAmelCase__ : List[str]   =						True

							lowerCAmelCase__ : Dict   =						["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
							lowerCAmelCase__ : Optional[Any]   =						"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

							# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
							lowerCAmelCase__ : Optional[int]   =						[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

							lowerCAmelCase__ : Union[str, Any]   =						PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
							tokenizer.save_pretrained(self.tmpdirname  )






				def 				_lowerCamelCase						(		self   : Any  ):






							'''simple docstring'''







							pass






				def 				_lowerCamelCase						(		self   : Optional[Any]  ):






							'''simple docstring'''







							pass






				def 				_lowerCamelCase						(		self   : Optional[Any]  ):






							'''simple docstring'''







							pass






				def 				_lowerCamelCase						(		self   : Dict  ):






							'''simple docstring'''







							pass






				def 				_lowerCamelCase						(		self   : Dict  ):






							'''simple docstring'''







							lowerCAmelCase__ : Optional[int]   =						self.tmpdirname
							# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
													try:
																lowerCAmelCase__ : int   =						tempfile.mkdtemp()
																lowerCAmelCase__ : str   =						self.rust_tokenizer_class.from_pretrained(snake_case__       ,	**snake_case__  )

																tokenizer.save_pretrained(self.tmpdirname  )
																super().test_training_new_tokenizer()
													finally:
																# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																# is restored
																shutil.rmtree(self.tmpdirname  )
																lowerCAmelCase__ : str   =						tmpdirname_orig






				def 				_lowerCamelCase						(		self   : Optional[int]  ):






							'''simple docstring'''







							lowerCAmelCase__ : List[Any]   =						self.tmpdirname
							# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
													try:
																lowerCAmelCase__ : Optional[int]   =						tempfile.mkdtemp()
																lowerCAmelCase__ : List[Any]   =						self.rust_tokenizer_class.from_pretrained(snake_case__       ,	**snake_case__  )

																tokenizer.save_pretrained(self.tmpdirname  )
																super().test_training_new_tokenizer_with_special_tokens_change()
													finally:
																# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																# is restored
																shutil.rmtree(self.tmpdirname  )
																lowerCAmelCase__ : Optional[int]   =						tmpdirname_orig






				def 				_lowerCamelCase						(		self   : Tuple  ):






							'''simple docstring'''







							lowerCAmelCase__ : str   =						self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

							lowerCAmelCase__ : Any   =						("""a""" for _ in range(1_000  ))
							lowerCAmelCase__ : Tuple   =						tokenizer.train_new_from_iterator(text_iterator=snake_case__       ,	length=1_000       ,	vocab_size=50  )

							lowerCAmelCase__ : Any   =						new_tokenizer.encode('a🤗'  )
							self.assertEqual(snake_case__       ,	[64, 172, 253, 97, 245]  )







@require_tokenizers
class 	A__				(	unittest.TestCase      ):






				def 				_lowerCamelCase						(		self   : Any  ):






							'''simple docstring'''







							lowerCAmelCase__ : List[str]   =						AutoTokenizer.from_pretrained('bert-base-cased'  )
							lowerCAmelCase__ : int   =						json.loads(tokenizer._tokenizer.to_str()  )
							lowerCAmelCase__ : Dict   =						len(snake_case__  )

							with tempfile.TemporaryDirectory() as tmp_dir:
										# Hack to save this in the tokenizer_config.json
										lowerCAmelCase__ : str   =						["""tokenizer.4.0.0.json"""]
										tokenizer.save_pretrained(snake_case__  )
										json.dump(snake_case__       ,	open(os.path.join(snake_case__       ,	'tokenizer.4.0.0.json'  )       ,	'w'  )  )

										# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
										lowerCAmelCase__ : List[Any]   =						AutoTokenizer.from_pretrained(snake_case__  )
										self.assertEqual(len(snake_case__  )       ,	len(snake_case__  ) + 1  )
										lowerCAmelCase__ : Dict   =						json.loads(new_tokenizer._tokenizer.to_str()  )
										self.assertIn('huggingface'       ,	json_tokenizer['model']['vocab']  )

										# Will need to be adjusted if we reach v42 and this test is still here.
										# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
										shutil.move(os.path.join(snake_case__       ,	'tokenizer.4.0.0.json'  )       ,	os.path.join(snake_case__       ,	'tokenizer.42.0.0.json'  )  )
										lowerCAmelCase__ : Tuple   =						["""tokenizer.42.0.0.json"""]
										tokenizer.save_pretrained(snake_case__  )
										lowerCAmelCase__ : Union[str, Any]   =						AutoTokenizer.from_pretrained(snake_case__  )
										self.assertEqual(len(snake_case__  )       ,	len(snake_case__  )  )
										lowerCAmelCase__ : List[Any]   =						json.loads(new_tokenizer._tokenizer.to_str()  )
										self.assertNotIn('huggingface'       ,	json_tokenizer['model']['vocab']  )






				def 				_lowerCamelCase						(		self   : str  ):






							'''simple docstring'''







							lowerCAmelCase__ : Optional[int]   =						"""hf-internal-testing/test-two-tokenizers"""

							# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
							lowerCAmelCase__ : str   =						AutoTokenizer.from_pretrained(snake_case__  )
							self.assertEqual(len(snake_case__  )       ,	28_997  )
							lowerCAmelCase__ : Dict   =						json.loads(tokenizer._tokenizer.to_str()  )
							self.assertIn('huggingface'       ,	json_tokenizer['model']['vocab']  )

							# Testing an older version by monkey-patching the version in the module it's used.
							import transformers as old_transformers

							lowerCAmelCase__ : Optional[Any]   =						"""3.0.0"""
							lowerCAmelCase__ : Tuple   =						old_transformers.models.auto.AutoTokenizer.from_pretrained(snake_case__  )
							self.assertEqual(len(snake_case__  )       ,	28_996  )
							lowerCAmelCase__ : Optional[int]   =						json.loads(old_tokenizer._tokenizer.to_str()  )
							self.assertNotIn('huggingface'       ,	json_tokenizer['model']['vocab']  )







@require_tokenizers
class 	A__				(	unittest.TestCase      ):






				def 				_lowerCamelCase						(		self   : Any  ):






							'''simple docstring'''







							lowerCAmelCase__ : Tuple   =						PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'  )
							lowerCAmelCase__ : Union[str, Any]   =						"""The Matrix is a 1999 science fiction action film."""

							with concurrent.futures.ThreadPoolExecutor() as executor:
										lowerCAmelCase__ : Any   =						[executor.submit(self.fetch       ,	snake_case__       ,	snake_case__  ) for i in range(10  )]
										lowerCAmelCase__ : List[str]   =						[future.result() for future in futures]
										self.assertEqual(snake_case__       ,	[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )






				def 				_lowerCamelCase						(		self   : List[Any]       ,	a   : Tuple       ,	a   : Dict  ):






							'''simple docstring'''







							return tokenizer.encode(snake_case__       ,	truncation='longest_first'       ,	padding='longest'  )

"""simple docstring"""
from transformers import ProcessorMixin




class  _snake_case	(    _lowercase							):
							SCREAMING_SNAKE_CASE		  =    '''AutoFeatureExtractor'''
							SCREAMING_SNAKE_CASE		  =    '''AutoTokenizer'''
