
'''simple docstring'''





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 			lowerCAmelCase__ (							UpperCamelCase       ,				unittest.TestCase							):



  '''simple docstring'''






  _UpperCamelCase							=			PreTrainedTokenizerFast
  _UpperCamelCase							=			False
  _UpperCamelCase							=			True
  _UpperCamelCase							=			'tokenizer_file'

  def 		_lowerCamelCase						(self	:			Any     ) ->							str:
        snake_case_    							=				False  # because we don't have pretrained_vocab_files_map
        super().setUp()
        snake_case_    							=				True

        snake_case_    							=				["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
        snake_case_    							=				"SaulLu/dummy-tokenizer-bytelevel-bpe"

        # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
        snake_case_    							=				[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

        snake_case_    							=				PreTrainedTokenizerFast.from_pretrained(model_paths[0]     )
        tokenizer.save_pretrained(self.tmpdirname     )

  def 		_lowerCamelCase						(self	:			Any     ) ->							List[Any]:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass

  def 		_lowerCamelCase						(self	:			Tuple     ) ->							Dict:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass

  def 		_lowerCamelCase						(self	:			Any     ) ->							Dict:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass

  def 		_lowerCamelCase						(self	:			Any     ) ->							Tuple:
        # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
        pass

  def 		_lowerCamelCase						(self	:			List[str]     ) ->							List[str]:
        snake_case_    							=				self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
                    try:
                          snake_case_    							=				tempfile.mkdtemp()
                          snake_case_    							=				self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_						,			**lowerCAmelCase_     )

                          tokenizer.save_pretrained(self.tmpdirname     )
                          super().test_training_new_tokenizer()
                    finally:
                          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                          # is restored
                          shutil.rmtree(self.tmpdirname     )
                          snake_case_    							=				tmpdirname_orig

  def 		_lowerCamelCase						(self	:			Union[str, Any]     ) ->							Dict:
        snake_case_    							=				self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
                    try:
                          snake_case_    							=				tempfile.mkdtemp()
                          snake_case_    							=				self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_						,			**lowerCAmelCase_     )

                          tokenizer.save_pretrained(self.tmpdirname     )
                          super().test_training_new_tokenizer_with_special_tokens_change()
                    finally:
                          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                          # is restored
                          shutil.rmtree(self.tmpdirname     )
                          snake_case_    							=				tmpdirname_orig


  def 		_lowerCamelCase						(self	:			Union[str, Any]     ) ->							Union[str, Any]:
        snake_case_    							=				self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name     )

        snake_case_    							=				("a" for _ in range(10_00     ))
        snake_case_    							=				tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase_						,			length=10_00						,			vocab_size=50     )

        snake_case_    							=				new_tokenizer.encode("a🤗"     )
        self.assertEqual(lowerCAmelCase_						,			[64, 1_72, 2_53, 97, 2_45]     )



@require_tokenizers
class 			lowerCAmelCase__ (							unittest.TestCase							):



  '''simple docstring'''






  def 		_lowerCamelCase						(self	:			Tuple     ) ->							Dict:
        snake_case_    							=				AutoTokenizer.from_pretrained("bert-base-cased"     )
        snake_case_    							=				json.loads(tokenizer._tokenizer.to_str()     )
        snake_case_    							=				len(lowerCAmelCase_     )

        with tempfile.TemporaryDirectory() as tmp_dir:
              # Hack to save this in the tokenizer_config.json
              snake_case_    							=				["tokenizer.4.0.0.json"]
              tokenizer.save_pretrained(lowerCAmelCase_     )
              json.dump(lowerCAmelCase_						,			open(os.path.join(lowerCAmelCase_						,			"tokenizer.4.0.0.json"     )						,			"w"     )     )

              # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
              snake_case_    							=				AutoTokenizer.from_pretrained(lowerCAmelCase_     )
              self.assertEqual(len(lowerCAmelCase_     )						,			len(lowerCAmelCase_     ) + 1     )
              snake_case_    							=				json.loads(new_tokenizer._tokenizer.to_str()     )
              self.assertIn("huggingface"						,			json_tokenizer["model"]["vocab"]     )

              # Will need to be adjusted if we reach v42 and this test is still here.
              # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
              shutil.move(os.path.join(lowerCAmelCase_						,			"tokenizer.4.0.0.json"     )						,			os.path.join(lowerCAmelCase_						,			"tokenizer.42.0.0.json"     )     )
              snake_case_    							=				["tokenizer.42.0.0.json"]
              tokenizer.save_pretrained(lowerCAmelCase_     )
              snake_case_    							=				AutoTokenizer.from_pretrained(lowerCAmelCase_     )
              self.assertEqual(len(lowerCAmelCase_     )						,			len(lowerCAmelCase_     )     )
              snake_case_    							=				json.loads(new_tokenizer._tokenizer.to_str()     )
              self.assertNotIn("huggingface"						,			json_tokenizer["model"]["vocab"]     )


  def 		_lowerCamelCase						(self	:			List[str]     ) ->							List[Any]:
        # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
        snake_case_    							=				"hf-internal-testing/test-two-tokenizers"

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        snake_case_    							=				AutoTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertEqual(len(lowerCAmelCase_     )						,			2_89_97     )
        snake_case_    							=				json.loads(tokenizer._tokenizer.to_str()     )
        self.assertIn("huggingface"						,			json_tokenizer["model"]["vocab"]     )

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        snake_case_    							=				"3.0.0"
        snake_case_    							=				old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertEqual(len(lowerCAmelCase_     )						,			2_89_96     )
        snake_case_    							=				json.loads(old_tokenizer._tokenizer.to_str()     )
        self.assertNotIn("huggingface"						,			json_tokenizer["model"]["vocab"]     )



@require_tokenizers
class 			lowerCAmelCase__ (							unittest.TestCase							):



  '''simple docstring'''






  def 		_lowerCamelCase						(self	:			List[Any]     ) ->							List[str]:
        # See https://github.com/huggingface/transformers/pull/12550
        # and https://github.com/huggingface/tokenizers/issues/537
        snake_case_    							=				PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"     )
        snake_case_    							=				"The Matrix is a 1999 science fiction action film."

        with concurrent.futures.ThreadPoolExecutor() as executor:
              snake_case_    							=				[executor.submit(self.fetch						,			lowerCAmelCase_						,			lowerCAmelCase_     ) for i in range(10     )]
              snake_case_    							=				[future.result() for future in futures]
              self.assertEqual(lowerCAmelCase_						,			[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10     )]     )


  def 		_lowerCamelCase						(self	:			Dict						,			lowerCAmelCase_	:			Optional[int]						,			lowerCAmelCase_	:			Optional[Any]     ) ->							Union[str, Any]:
        return tokenizer.encode(lowerCAmelCase_						,			truncation="longest_first"						,			padding="longest"     )





'''simple docstring'''

from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

import numpy as np
import PIL
from PIL import Image

from ...utils import BaseOutput, is_torch_available, is_transformers_available


@dataclass
class    a__					(			__A   ):







				"""simple docstring"""



				__UpperCamelCase     :		Tuple						=						{
				    'sld_warmup_steps': 15,
				    'sld_guidance_scale': 20,
				    'sld_threshold': 0.0,
				    'sld_momentum_scale': 0.0,
				    'sld_mom_beta': 0.0,
				}
				__UpperCamelCase     :		Optional[Any]						=						{
				    'sld_warmup_steps': 10,
				    'sld_guidance_scale': 1_000,
				    'sld_threshold': 0.01,
				    'sld_momentum_scale': 0.3,
				    'sld_mom_beta': 0.4,
				}
				__UpperCamelCase     :		Tuple						=						{
				    'sld_warmup_steps': 7,
				    'sld_guidance_scale': 2_000,
				    'sld_threshold': 0.025,
				    'sld_momentum_scale': 0.5,
				    'sld_mom_beta': 0.7,
				}
				__UpperCamelCase     :		str						=						{
				    'sld_warmup_steps': 0,
				    'sld_guidance_scale': 5_000,
				    'sld_threshold': 1.0,
				    'sld_momentum_scale': 0.5,
				    'sld_mom_beta': 0.7,
				}


@dataclass
class    a__					(			__A   ):







				"""simple docstring"""



				__UpperCamelCase     :		Union[List[PIL.Image.Image], np.ndarray]
				__UpperCamelCase     :		Optional[List[bool]]
				__UpperCamelCase     :		Optional[Union[List[PIL.Image.Image], np.ndarray]]
				__UpperCamelCase     :		Optional[str]


if is_transformers_available() and is_torch_available():
	from .pipeline_stable_diffusion_safe import StableDiffusionPipelineSafe
	from .safety_checker import SafeStableDiffusionSafetyChecker

