
"""simple docstring"""
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  _snake_case	(    _lowercase			,					unittest.TestCase							):
							SCREAMING_SNAKE_CASE		  =    PreTrainedTokenizerFast
							SCREAMING_SNAKE_CASE		  =    False
							SCREAMING_SNAKE_CASE		  =    True
							SCREAMING_SNAKE_CASE		  =    '''tokenizer_file'''






							def 				__lowercase	(       self    :      int     ):






												"""simple docstring"""



												__a:    Union[str, Any]		     =       False  # because we don't have pretrained_vocab_files_map
												super().setUp()
												__a:    List[str]		     =       True

												__a:    Dict		     =       ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
												__a:    Optional[Any]		     =       """SaulLu/dummy-tokenizer-bytelevel-bpe"""

												# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
												__a:    Optional[int]		     =       [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

												__a:    Union[str, Any]		     =       PreTrainedTokenizerFast.from_pretrained(model_paths[0]     )
												tokenizer.save_pretrained(self.tmpdirname     )






							def 				__lowercase	(       self    :      Any     ):






												"""simple docstring"""



												pass






							def 				__lowercase	(       self    :      Optional[Any]     ):






												"""simple docstring"""



												pass






							def 				__lowercase	(       self    :      Optional[Any]     ):






												"""simple docstring"""



												pass






							def 				__lowercase	(       self    :      Dict     ):






												"""simple docstring"""



												pass






							def 				__lowercase	(       self    :      Dict     ):






												"""simple docstring"""



												__a:    Optional[int]		     =       self.tmpdirname
												# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
																						try:
																											__a:    int		     =       tempfile.mkdtemp()
																											__a:    str		     =       self.rust_tokenizer_class.from_pretrained(snake_case__    ,			**snake_case__     )

																											tokenizer.save_pretrained(self.tmpdirname     )
																											super().test_training_new_tokenizer()
																						finally:
																											# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																											# is restored
																											shutil.rmtree(self.tmpdirname     )
																											__a:    str		     =       tmpdirname_orig






							def 				__lowercase	(       self    :      Optional[int]     ):






												"""simple docstring"""



												__a:    List[Any]		     =       self.tmpdirname
												# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
																						try:
																											__a:    Optional[int]		     =       tempfile.mkdtemp()
																											__a:    List[Any]		     =       self.rust_tokenizer_class.from_pretrained(snake_case__    ,			**snake_case__     )

																											tokenizer.save_pretrained(self.tmpdirname     )
																											super().test_training_new_tokenizer_with_special_tokens_change()
																						finally:
																											# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																											# is restored
																											shutil.rmtree(self.tmpdirname     )
																											__a:    Optional[int]		     =       tmpdirname_orig






							def 				__lowercase	(       self    :      Tuple     ):






												"""simple docstring"""



												__a:    str		     =       self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name     )

												__a:    Any		     =       ("""a""" for _ in range(1_0_0_0     ))
												__a:    Tuple		     =       tokenizer.train_new_from_iterator(text_iterator=snake_case__    ,			length=1_0_0_0    ,			vocab_size=5_0     )

												__a:    Any		     =       new_tokenizer.encode("""a🤗"""     )
												self.assertEqual(snake_case__    ,			[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]     )






@require_tokenizers
class  _snake_case	(    unittest.TestCase							):






							def 				__lowercase	(       self    :      Any     ):






												"""simple docstring"""



												__a:    List[str]		     =       AutoTokenizer.from_pretrained("""bert-base-cased"""     )
												__a:    int		     =       json.loads(tokenizer._tokenizer.to_str()     )
												__a:    Dict		     =       len(snake_case__     )

												with tempfile.TemporaryDirectory() as tmp_dir:
																	# Hack to save this in the tokenizer_config.json
																	__a:    str		     =       ["""tokenizer.4.0.0.json"""]
																	tokenizer.save_pretrained(snake_case__     )
																	json.dump(snake_case__    ,			open(os.path.join(snake_case__    ,			"""tokenizer.4.0.0.json"""     )    ,			"""w"""     )     )

																	# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
																	__a:    List[Any]		     =       AutoTokenizer.from_pretrained(snake_case__     )
																	self.assertEqual(len(snake_case__     )    ,			len(snake_case__     ) + 1     )
																	__a:    Dict		     =       json.loads(new_tokenizer._tokenizer.to_str()     )
																	self.assertIn("""huggingface"""    ,			json_tokenizer["""model"""]["""vocab"""]     )

																	# Will need to be adjusted if we reach v42 and this test is still here.
																	# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
																	shutil.move(os.path.join(snake_case__    ,			"""tokenizer.4.0.0.json"""     )    ,			os.path.join(snake_case__    ,			"""tokenizer.42.0.0.json"""     )     )
																	__a:    Tuple		     =       ["""tokenizer.42.0.0.json"""]
																	tokenizer.save_pretrained(snake_case__     )
																	__a:    Union[str, Any]		     =       AutoTokenizer.from_pretrained(snake_case__     )
																	self.assertEqual(len(snake_case__     )    ,			len(snake_case__     )     )
																	__a:    List[Any]		     =       json.loads(new_tokenizer._tokenizer.to_str()     )
																	self.assertNotIn("""huggingface"""    ,			json_tokenizer["""model"""]["""vocab"""]     )






							def 				__lowercase	(       self    :      str     ):






												"""simple docstring"""



												__a:    Optional[int]		     =       """hf-internal-testing/test-two-tokenizers"""

												# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
												__a:    str		     =       AutoTokenizer.from_pretrained(snake_case__     )
												self.assertEqual(len(snake_case__     )    ,			2_8_9_9_7     )
												__a:    Dict		     =       json.loads(tokenizer._tokenizer.to_str()     )
												self.assertIn("""huggingface"""    ,			json_tokenizer["""model"""]["""vocab"""]     )

												# Testing an older version by monkey-patching the version in the module it's used.
												import transformers as old_transformers

												__a:    Optional[Any]		     =       """3.0.0"""
												__a:    Tuple		     =       old_transformers.models.auto.AutoTokenizer.from_pretrained(snake_case__     )
												self.assertEqual(len(snake_case__     )    ,			2_8_9_9_6     )
												__a:    Optional[int]		     =       json.loads(old_tokenizer._tokenizer.to_str()     )
												self.assertNotIn("""huggingface"""    ,			json_tokenizer["""model"""]["""vocab"""]     )






@require_tokenizers
class  _snake_case	(    unittest.TestCase							):






							def 				__lowercase	(       self    :      Any     ):






												"""simple docstring"""



												__a:    Tuple		     =       PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""     )
												__a:    Union[str, Any]		     =       """The Matrix is a 1999 science fiction action film."""

												with concurrent.futures.ThreadPoolExecutor() as executor:
																	__a:    Any		     =       [executor.submit(self.fetch    ,			snake_case__    ,			snake_case__     ) for i in range(1_0     )]
																	__a:    List[str]		     =       [future.result() for future in futures]
																	self.assertEqual(snake_case__    ,			[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0     )]     )






							def 				__lowercase	(       self    :      List[Any]    ,			snake_case__    :      Tuple    ,			snake_case__    :      Dict     ):






												"""simple docstring"""



												return tokenizer.encode(snake_case__    ,			truncation="""longest_first"""    ,			padding="""longest"""     )


"""simple docstring"""
import unittest
from dataclasses import dataclass
from typing import Optional

from transformers.utils import ModelOutput




@dataclass
class  _snake_case	(    _lowercase							):
							SCREAMING_SNAKE_CASE		  =    42
							SCREAMING_SNAKE_CASE		  =    None
							SCREAMING_SNAKE_CASE		  =    None






class  _snake_case	(    unittest.TestCase							):






							def 				__lowercase	(       self    :      Optional[Any]     ):






												"""simple docstring"""



												__a:    Dict		     =       ModelOutputTest(a=3_0     )
												self.assertEqual(x.a    ,			3_0     )
												self.assertIsNone(x.b     )
												self.assertIsNone(x.c     )
												with self.assertRaises(snake_case__     ):
																	__a:    int		     =       x.d






							def 				__lowercase	(       self    :      int     ):






												"""simple docstring"""



												__a:    Tuple		     =       ModelOutputTest(a=3_0    ,			b=1_0     )
												self.assertEqual(x[0]    ,			3_0     )
												self.assertEqual(x[1]    ,			1_0     )
												self.assertEqual(x[:2]    ,			(3_0, 1_0)     )
												self.assertEqual(x[:]    ,			(3_0, 1_0)     )

												__a:    Tuple		     =       ModelOutputTest(a=3_0    ,			c=1_0     )
												self.assertEqual(x[0]    ,			3_0     )
												self.assertEqual(x[1]    ,			1_0     )
												self.assertEqual(x[:2]    ,			(3_0, 1_0)     )
												self.assertEqual(x[:]    ,			(3_0, 1_0)     )






							def 				__lowercase	(       self    :      Tuple     ):






												"""simple docstring"""



												__a:    List[str]		     =       ModelOutputTest(a=3_0    ,			b=1_0     )
												self.assertEqual(x["""a"""]    ,			3_0     )
												self.assertEqual(x["""b"""]    ,			1_0     )
												with self.assertRaises(snake_case__     ):
																	__a:    Union[str, Any]		     =       x["""c"""]

												__a:    str		     =       ModelOutputTest(a=3_0    ,			c=1_0     )
												self.assertEqual(x["""a"""]    ,			3_0     )
												self.assertEqual(x["""c"""]    ,			1_0     )
												with self.assertRaises(snake_case__     ):
																	__a:    List[str]		     =       x["""b"""]






							def 				__lowercase	(       self    :      List[str]     ):






												"""simple docstring"""



												__a:    str		     =       ModelOutputTest(a=3_0     )
												self.assertEqual(list(x.keys()     )    ,			["""a"""]     )
												self.assertEqual(list(x.values()     )    ,			[3_0]     )
												self.assertEqual(list(x.items()     )    ,			[("""a""", 3_0)]     )
												self.assertEqual(list(snake_case__     )    ,			["""a"""]     )

												__a:    Optional[Any]		     =       ModelOutputTest(a=3_0    ,			b=1_0     )
												self.assertEqual(list(x.keys()     )    ,			["""a""", """b"""]     )
												self.assertEqual(list(x.values()     )    ,			[3_0, 1_0]     )
												self.assertEqual(list(x.items()     )    ,			[("""a""", 3_0), ("""b""", 1_0)]     )
												self.assertEqual(list(snake_case__     )    ,			["""a""", """b"""]     )

												__a:    Union[str, Any]		     =       ModelOutputTest(a=3_0    ,			c=1_0     )
												self.assertEqual(list(x.keys()     )    ,			["""a""", """c"""]     )
												self.assertEqual(list(x.values()     )    ,			[3_0, 1_0]     )
												self.assertEqual(list(x.items()     )    ,			[("""a""", 3_0), ("""c""", 1_0)]     )
												self.assertEqual(list(snake_case__     )    ,			["""a""", """c"""]     )

												with self.assertRaises(snake_case__     ):
																	__a:    List[Any]		     =       x.update({"""d""": 2_0}     )
												with self.assertRaises(snake_case__     ):
																	del x["a"]
												with self.assertRaises(snake_case__     ):
																	__a:    Union[str, Any]		     =       x.pop("""a"""     )
												with self.assertRaises(snake_case__     ):
																	__a:    List[Any]		     =       x.setdefault("""d"""    ,			3_2     )






							def 				__lowercase	(       self    :      Any     ):






												"""simple docstring"""



												__a:    List[str]		     =       ModelOutputTest(a=3_0     )
												__a:    str		     =       1_0
												self.assertEqual(x.a    ,			1_0     )
												self.assertEqual(x["""a"""]    ,			1_0     )






							def 				__lowercase	(       self    :      Dict     ):






												"""simple docstring"""



												__a:    List[str]		     =       ModelOutputTest(a=3_0     )
												__a:    Any		     =       1_0
												self.assertEqual(x.a    ,			1_0     )
												self.assertEqual(x["""a"""]    ,			1_0     )






							def 				__lowercase	(       self    :      int     ):






												"""simple docstring"""



												__a:    List[Any]		     =       ModelOutputTest({"""a""": 3_0, """b""": 1_0}     )
												self.assertEqual(list(x.keys()     )    ,			["""a""", """b"""]     )
												self.assertEqual(x.a    ,			3_0     )
												self.assertEqual(x.b    ,			1_0     )






							def 				__lowercase	(       self    :      List[str]     ):






												"""simple docstring"""



												__a:    Union[str, Any]		     =       ModelOutputTest([("""a""", 3_0), ("""b""", 1_0)]     )
												self.assertEqual(list(x.keys()     )    ,			["""a""", """b"""]     )
												self.assertEqual(x.a    ,			3_0     )
												self.assertEqual(x.b    ,			1_0     )

												with self.assertRaises(snake_case__     ):
																	__a:    str		     =       ModelOutputTest([("""a""", 3_0), (1_0, 1_0)]     )

												__a:    str		     =       ModelOutputTest(a=(3_0, 3_0)     )
												self.assertEqual(list(x.keys()     )    ,			["""a"""]     )
												self.assertEqual(x.a    ,			(3_0, 3_0)     )
