


"""simple docstring"""






import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 			SCREAMING_SNAKE_CASE_  (   _lowercase	,      unittest.TestCase):




							'''simple docstring'''





							__magic_name__					:     int				  =  PreTrainedTokenizerFast
							__magic_name__					:     Tuple				  =  False
							__magic_name__					:     str				  =  True
							__magic_name__					:     Tuple				  =  '''tokenizer_file'''




							def        UpperCAmelCase (						self)   ->	Dict:







								'''simple docstring'''




								snake_case__  :						Union[str, Any]		=      False  # because we don't have pretrained_vocab_files_map
								super().setUp()
								snake_case__  :						Any		=      True

								snake_case__  :						List[str]		=      ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
								snake_case__  :						List[str]		=      "SaulLu/dummy-tokenizer-bytelevel-bpe"

								# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
								snake_case__  :						Tuple		=      [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

								snake_case__  :						Union[str, Any]		=      PreTrainedTokenizerFast.from_pretrained(model_paths[0])
								tokenizer.save_pretrained(self.tmpdirname)




							def        UpperCAmelCase (						self)   ->	Optional[int]:







								'''simple docstring'''




								pass




							def        UpperCAmelCase (						self)   ->	Optional[int]:







								'''simple docstring'''




								pass




							def        UpperCAmelCase (						self)   ->	Union[str, Any]:







								'''simple docstring'''




								pass




							def        UpperCAmelCase (						self)   ->	str:







								'''simple docstring'''




								pass




							def        UpperCAmelCase (						self)   ->	List[Any]:







								'''simple docstring'''




								snake_case__  :						Optional[Any]		=      self.tmpdirname
								# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
										try:
											snake_case__  :						List[str]		=      tempfile.mkdtemp()
											snake_case__  :						Optional[Any]		=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,					**lowerCamelCase__)

											tokenizer.save_pretrained(self.tmpdirname)
											super().test_training_new_tokenizer()
										finally:
											# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
											# is restored
											shutil.rmtree(self.tmpdirname)
											snake_case__  :						List[Any]		=      tmpdirname_orig




							def        UpperCAmelCase (						self)   ->	Dict:







								'''simple docstring'''




								snake_case__  :						Tuple		=      self.tmpdirname
								# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
										try:
											snake_case__  :						Optional[int]		=      tempfile.mkdtemp()
											snake_case__  :						str		=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,					**lowerCamelCase__)

											tokenizer.save_pretrained(self.tmpdirname)
											super().test_training_new_tokenizer_with_special_tokens_change()
										finally:
											# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
											# is restored
											shutil.rmtree(self.tmpdirname)
											snake_case__  :						int		=      tmpdirname_orig







							def        UpperCAmelCase (						self)   ->	int:







								'''simple docstring'''




								snake_case__  :						str		=      self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name)

								snake_case__  :						Dict		=      ("a" for _ in range(1_000))
								snake_case__  :						Any		=      tokenizer.train_new_from_iterator(text_iterator=lowerCamelCase__	,					length=1_000	,					vocab_size=50)

								snake_case__  :						Optional[Any]		=      new_tokenizer.encode("a🤗")
								self.assertEqual(lowerCamelCase__	,					[64, 172, 253, 97, 245])


@require_tokenizers
class 			SCREAMING_SNAKE_CASE_  (   unittest.TestCase):




							'''simple docstring'''





							def        UpperCAmelCase (						self)   ->	List[str]:







								'''simple docstring'''




								snake_case__  :						List[Any]		=      AutoTokenizer.from_pretrained("bert-base-cased")
								snake_case__  :						Dict		=      json.loads(tokenizer._tokenizer.to_str())
								snake_case__  :						Any		=      len(lowerCamelCase__)

								with tempfile.TemporaryDirectory() as tmp_dir:
									# Hack to save this in the tokenizer_config.json
									snake_case__  :						List[Any]		=      ["tokenizer.4.0.0.json"]
									tokenizer.save_pretrained(lowerCamelCase__)
									json.dump(lowerCamelCase__	,					open(os.path.join(lowerCamelCase__	,					"tokenizer.4.0.0.json")	,					"w"))

									# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
									snake_case__  :						Tuple		=      AutoTokenizer.from_pretrained(lowerCamelCase__)
									self.assertEqual(len(lowerCamelCase__)	,					len(lowerCamelCase__) + 1)
									snake_case__  :						Union[str, Any]		=      json.loads(new_tokenizer._tokenizer.to_str())
									self.assertIn("huggingface"	,					json_tokenizer["model"]["vocab"])

									# Will need to be adjusted if we reach v42 and this test is still here.
									# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
									shutil.move(os.path.join(lowerCamelCase__	,					"tokenizer.4.0.0.json")	,					os.path.join(lowerCamelCase__	,					"tokenizer.42.0.0.json"))
									snake_case__  :						Optional[int]		=      ["tokenizer.42.0.0.json"]
									tokenizer.save_pretrained(lowerCamelCase__)
									snake_case__  :						List[str]		=      AutoTokenizer.from_pretrained(lowerCamelCase__)
									self.assertEqual(len(lowerCamelCase__)	,					len(lowerCamelCase__))
									snake_case__  :						Any		=      json.loads(new_tokenizer._tokenizer.to_str())
									self.assertNotIn("huggingface"	,					json_tokenizer["model"]["vocab"])







							def        UpperCAmelCase (						self)   ->	Union[str, Any]:







								'''simple docstring'''




								snake_case__  :						List[Any]		=      "hf-internal-testing/test-two-tokenizers"

								# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
								snake_case__  :						Union[str, Any]		=      AutoTokenizer.from_pretrained(lowerCamelCase__)
								self.assertEqual(len(lowerCamelCase__)	,					28_997)
								snake_case__  :						Optional[Any]		=      json.loads(tokenizer._tokenizer.to_str())
								self.assertIn("huggingface"	,					json_tokenizer["model"]["vocab"])

								# Testing an older version by monkey-patching the version in the module it's used.
								import transformers as old_transformers

								snake_case__  :						Any		=      "3.0.0"
								snake_case__  :						Optional[Any]		=      old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCamelCase__)
								self.assertEqual(len(lowerCamelCase__)	,					28_996)
								snake_case__  :						Any		=      json.loads(old_tokenizer._tokenizer.to_str())
								self.assertNotIn("huggingface"	,					json_tokenizer["model"]["vocab"])


@require_tokenizers
class 			SCREAMING_SNAKE_CASE_  (   unittest.TestCase):




							'''simple docstring'''





							def        UpperCAmelCase (						self)   ->	Union[str, Any]:







								'''simple docstring'''




								snake_case__  :						str		=      PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel")
								snake_case__  :						Any		=      "The Matrix is a 1999 science fiction action film."

								with concurrent.futures.ThreadPoolExecutor() as executor:
									snake_case__  :						Dict		=      [executor.submit(self.fetch	,					lowerCamelCase__	,					lowerCamelCase__) for i in range(10)]
									snake_case__  :						Dict		=      [future.result() for future in futures]
									self.assertEqual(lowerCamelCase__	,					[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10)])







							def        UpperCAmelCase (						self	,					lowerCamelCase__	,					lowerCamelCase__)   ->	Union[str, Any]:







								'''simple docstring'''




								return tokenizer.encode(lowerCamelCase__	,					truncation="longest_first"	,					padding="longest")







"""simple docstring"""






import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 							snake_case       (     SCREAMING_SNAKE_CASE_							,       unittest.TestCase		):
   a_					:		Optional[int]     					=     GPTNeoXJapaneseTokenizer
   a_					:		Dict     					=     False
   a_					:		Union[str, Any]     					=     {"""do_clean_text""": False, """add_prefix_space""": False}







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      super().setUp()

      a_						   =     [
          "こん",
          "こんに",
          "にちは",
          "ばんは",
          "世界,㔺界",
          "、",
          "。",
          "<BR>",
          "<SP>",
          "<TAB>",
          "<URL>",
          "<EMAIL>",
          "<TEL>",
          "<DATE>",
          "<PRICE>",
          "<BLOCK>",
          "<KIGOU>",
          "<U2000U2BFF>",
          "<|emoji1|>",
          "<unk>",
          "<|startoftext|>",
          "<|endoftext|>",
      ]
      a_						   =     {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
      a_						   =     {"unk_token": "<unk>"}

      a_						   =     os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["vocab_file"])
      a_						   =     os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["emoji_file"])
      with open(self.vocab_file       ,							"w"       ,							encoding="utf-8") as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
      with open(self.emoji_file       ,							"w") as emoji_writer:
         emoji_writer.write(json.dumps(__UpperCAmelCase))







   def    UpperCAmelCase__	( self       ,							**__UpperCAmelCase)   ->int:
      kwargs.update(self.special_tokens_map)
      return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname       ,							**__UpperCAmelCase)







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Dict:
      a_						   =     "こんにちは、世界。 \nこんばんは、㔺界。😀"
      a_						   =     "こんにちは、世界。 \nこんばんは、世界。😀"
      return input_text, output_text







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Optional[Any]:
      a_   ,			a_						   =     self.get_input_output_texts(__UpperCAmelCase)
      a_						   =     tokenizer.encode(__UpperCAmelCase       ,							add_special_tokens=__UpperCAmelCase)
      a_						   =     tokenizer.decode(__UpperCAmelCase       ,							clean_up_tokenization_spaces=__UpperCAmelCase)
      return text, ids







   def    UpperCAmelCase__	( self)   ->Any:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->int:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->Optional[Any]:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->List[str]:
      a_						   =     self.get_tokenizer()

      # Testing tokenization
      a_						   =     "こんにちは、世界。　こんばんは、㔺界。"
      a_						   =     ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
      a_						   =     tokenizer.tokenize(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)

      # Testing conversion to ids without special tokens
      a_						   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
      a_						   =     tokenizer.convert_tokens_to_ids(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)

      # Testing conversion to ids with special tokens
      a_						   =     tokens + [tokenizer.unk_token]
      a_						   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
      a_						   =     tokenizer.convert_tokens_to_ids(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   @slow
   def    UpperCAmelCase__	( self)   ->Tuple:
      a_						   =     self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b")

      a_						   =     tokenizer.encode("ありがとう。"       ,							add_special_tokens=__UpperCAmelCase)
      a_						   =     tokenizer.encode("どういたしまして。"       ,							add_special_tokens=__UpperCAmelCase)

      a_						   =     tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase)
      a_						   =     tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase       ,							__UpperCAmelCase)

      assert encoded_sentence == ids_a
      assert encoded_pair == ids_a + ids_a







   def    UpperCAmelCase__	( self)   ->Tuple:
      # Intentionally convert some words to accommodate character fluctuations unique to Japanese
      pass







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      # tokenizer has no padding token
      pass