
"""simple docstring"""




import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 	UpperCamelCase      (			__magic_name__    , unittest.TestCase ):

				"""simple docstring"""



				UpperCAmelCase_							: int		    =							RealmTokenizer
				UpperCAmelCase_							: Union[str, Any]		    =							RealmTokenizerFast
				UpperCAmelCase_							: List[Any]		    =							True
				UpperCAmelCase_							: Union[str, Any]		    =							True
				UpperCAmelCase_							: Union[str, Any]		    =							filter_non_english


				def 				A   (							self     )       ->       Optional[int]:
									"""simple docstring"""







									super().setUp()

									SCREAMING_SNAKE_CASE           =      [
									    '[UNK]',
									    '[CLS]',
									    '[SEP]',
									    '[PAD]',
									    '[MASK]',
									    'want',
									    '##want',
									    '##ed',
									    'wa',
									    'un',
									    'runn',
									    '##ing',
									    ',',
									    'low',
									    'lowest',
									]
									SCREAMING_SNAKE_CASE           =      os.path.join(self.tmpdirname						,      VOCAB_FILES_NAMES['vocab_file']     )
									with open(self.vocab_file						,      'w'						,      encoding='utf-8'     ) as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )


				def 				A   (							self						,      lowercase__     )       ->       List[str]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      'UNwant\u00E9d,running'
									SCREAMING_SNAKE_CASE           =      'unwanted, running'
									return input_text, output_text


				def 				A   (							self     )       ->       List[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      self.tokenizer_class(self.vocab_file     )

									SCREAMING_SNAKE_CASE           =      tokenizer.tokenize('UNwant\u00E9d,running'     )
									self.assertListEqual(_A						,      ['un', '##want', '##ed', ',', 'runn', '##ing']     )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_A     )						,      [9, 6, 7, 12, 10, 11]     )


				def 				A   (							self     )       ->       Any:
									"""simple docstring"""







									if not self.test_rust_tokenizer:
														return

									SCREAMING_SNAKE_CASE           =      self.get_tokenizer()
									SCREAMING_SNAKE_CASE           =      self.get_rust_tokenizer()

									SCREAMING_SNAKE_CASE           =      'UNwant\u00E9d,running'

									SCREAMING_SNAKE_CASE           =      tokenizer.tokenize(_A     )
									SCREAMING_SNAKE_CASE           =      rust_tokenizer.tokenize(_A     )
									self.assertListEqual(_A						,      _A     )

									SCREAMING_SNAKE_CASE           =      tokenizer.encode(_A						,      add_special_tokens=_A     )
									SCREAMING_SNAKE_CASE           =      rust_tokenizer.encode(_A						,      add_special_tokens=_A     )
									self.assertListEqual(_A						,      _A     )

									SCREAMING_SNAKE_CASE           =      self.get_rust_tokenizer()
									SCREAMING_SNAKE_CASE           =      tokenizer.encode(_A     )
									SCREAMING_SNAKE_CASE           =      rust_tokenizer.encode(_A     )
									self.assertListEqual(_A						,      _A     )

									# With lower casing
									SCREAMING_SNAKE_CASE           =      self.get_tokenizer(do_lower_case=_A     )
									SCREAMING_SNAKE_CASE           =      self.get_rust_tokenizer(do_lower_case=_A     )

									SCREAMING_SNAKE_CASE           =      'UNwant\u00E9d,running'

									SCREAMING_SNAKE_CASE           =      tokenizer.tokenize(_A     )
									SCREAMING_SNAKE_CASE           =      rust_tokenizer.tokenize(_A     )
									self.assertListEqual(_A						,      _A     )

									SCREAMING_SNAKE_CASE           =      tokenizer.encode(_A						,      add_special_tokens=_A     )
									SCREAMING_SNAKE_CASE           =      rust_tokenizer.encode(_A						,      add_special_tokens=_A     )
									self.assertListEqual(_A						,      _A     )

									SCREAMING_SNAKE_CASE           =      self.get_rust_tokenizer()
									SCREAMING_SNAKE_CASE           =      tokenizer.encode(_A     )
									SCREAMING_SNAKE_CASE           =      rust_tokenizer.encode(_A     )
									self.assertListEqual(_A						,      _A     )


				def 				A   (							self     )       ->       List[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'     )						,      ['ah', '\u535A', '\u63A8', 'zz']     )


				def 				A   (							self     )       ->       Optional[int]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      BasicTokenizer(do_lower_case=_A     )

									self.assertListEqual(
									    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '     )						,      ['hello', '!', 'how', 'are', 'you', '?']     )
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )						,      ['hello']     )


				def 				A   (							self     )       ->       Tuple:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      BasicTokenizer(do_lower_case=_A						,      strip_accents=_A     )

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )						,      ['hällo', '!', 'how', 'are', 'you', '?']     )
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )						,      ['h\u00E9llo']     )


				def 				A   (							self     )       ->       Optional[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      BasicTokenizer(do_lower_case=_A						,      strip_accents=_A     )

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )						,      ['hallo', '!', 'how', 'are', 'you', '?']     )
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )						,      ['hello']     )


				def 				A   (							self     )       ->       Any:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      BasicTokenizer(do_lower_case=_A     )

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )						,      ['hallo', '!', 'how', 'are', 'you', '?']     )
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )						,      ['hello']     )


				def 				A   (							self     )       ->       int:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      BasicTokenizer(do_lower_case=_A     )

									self.assertListEqual(
									    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '     )						,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?']     )


				def 				A   (							self     )       ->       Any:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      BasicTokenizer(do_lower_case=_A						,      strip_accents=_A     )

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )						,      ['HäLLo', '!', 'how', 'Are', 'yoU', '?']     )


				def 				A   (							self     )       ->       Optional[int]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      BasicTokenizer(do_lower_case=_A						,      strip_accents=_A     )

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )						,      ['HaLLo', '!', 'how', 'Are', 'yoU', '?']     )


				def 				A   (							self     )       ->       Optional[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      BasicTokenizer(do_lower_case=_A						,      never_split=['[UNK]']     )

									self.assertListEqual(
									    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'     )						,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']     )


				def 				A   (							self     )       ->       Optional[int]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

									SCREAMING_SNAKE_CASE           =      {}
									for i, token in enumerate(_A     ):
														SCREAMING_SNAKE_CASE           =      i
									SCREAMING_SNAKE_CASE           =      WordpieceTokenizer(vocab=_A						,      unk_token='[UNK]'     )

									self.assertListEqual(tokenizer.tokenize(''     )						,      []     )

									self.assertListEqual(tokenizer.tokenize('unwanted running'     )						,      ['un', '##want', '##ed', 'runn', '##ing']     )

									self.assertListEqual(tokenizer.tokenize('unwantedX running'     )						,      ['[UNK]', 'runn', '##ing']     )


				def 				A   (							self     )       ->       Optional[Any]:
									"""simple docstring"""







									self.assertTrue(_is_whitespace(' '     )     )
									self.assertTrue(_is_whitespace('\t'     )     )
									self.assertTrue(_is_whitespace('\r'     )     )
									self.assertTrue(_is_whitespace('\n'     )     )
									self.assertTrue(_is_whitespace('\u00A0'     )     )

									self.assertFalse(_is_whitespace('A'     )     )
									self.assertFalse(_is_whitespace('-'     )     )


				def 				A   (							self     )       ->       Tuple:
									"""simple docstring"""







									self.assertTrue(_is_control('\u0005'     )     )

									self.assertFalse(_is_control('A'     )     )
									self.assertFalse(_is_control(' '     )     )
									self.assertFalse(_is_control('\t'     )     )
									self.assertFalse(_is_control('\r'     )     )


				def 				A   (							self     )       ->       Any:
									"""simple docstring"""







									self.assertTrue(_is_punctuation('-'     )     )
									self.assertTrue(_is_punctuation('$'     )     )
									self.assertTrue(_is_punctuation('`'     )     )
									self.assertTrue(_is_punctuation('.'     )     )

									self.assertFalse(_is_punctuation('A'     )     )
									self.assertFalse(_is_punctuation(' '     )     )


				def 				A   (							self     )       ->       Dict:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(_A     ) for t in ['Test', '\xad', 'test']]						,      [['[UNK]'], [], ['[UNK]']]     )

									if self.test_rust_tokenizer:
														SCREAMING_SNAKE_CASE           =      self.get_rust_tokenizer()
														self.assertListEqual(
														    [rust_tokenizer.tokenize(_A     ) for t in ['Test', '\xad', 'test']]						,      [['[UNK]'], [], ['[UNK]']]     )


				@slow
				def 				A   (							self     )       ->       List[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =      self.tokenizer_class.from_pretrained('bert-base-uncased'     )

									SCREAMING_SNAKE_CASE           =      tokenizer.encode('sequence builders'						,      add_special_tokens=_A     )
									SCREAMING_SNAKE_CASE           =      tokenizer.encode('multi-sequence build'						,      add_special_tokens=_A     )

									SCREAMING_SNAKE_CASE           =      tokenizer.build_inputs_with_special_tokens(_A     )
									SCREAMING_SNAKE_CASE           =      tokenizer.build_inputs_with_special_tokens(_A						,      _A     )

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]


				def 				A   (							self     )       ->       str:
									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
																			SCREAMING_SNAKE_CASE           =      self.rust_tokenizer_class.from_pretrained(_A						,      **_A     )

																			SCREAMING_SNAKE_CASE           =      f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																			SCREAMING_SNAKE_CASE           =      tokenizer_r.encode_plus(
																			    _A						,      return_attention_mask=_A						,      return_token_type_ids=_A						,      return_offsets_mapping=_A						,      add_special_tokens=_A						,      )

																			SCREAMING_SNAKE_CASE           =      tokenizer_r.do_lower_case if hasattr(_A						,      'do_lower_case'     ) else False
																			SCREAMING_SNAKE_CASE           =      (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'A'),
																			        ((1, 2), ','),
																			        ((3, 5), 'na'),
																			        ((5, 6), '##ï'),
																			        ((6, 8), '##ve'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'Allen'),
																			        ((21, 23), '##NL'),
																			        ((23, 24), '##P'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'a'),
																			        ((1, 2), ','),
																			        ((3, 8), 'naive'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'allen'),
																			        ((21, 23), '##nl'),
																			        ((23, 24), '##p'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]						,      tokenizer_r.convert_ids_to_tokens(tokens['input_ids']     )     )
																			self.assertEqual([e[0] for e in expected_results]						,      tokens['offset_mapping']     )




				@slow
				def 				A   (							self     )       ->       Tuple:
									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
																			SCREAMING_SNAKE_CASE           =      self.rust_tokenizer_class.from_pretrained(_A						,      **_A     )
																			SCREAMING_SNAKE_CASE           =      self.tokenizer_class.from_pretrained(_A						,      **_A     )
																			SCREAMING_SNAKE_CASE           =      [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

																			SCREAMING_SNAKE_CASE           =      tokenizer_r.batch_encode_candidates(_A						,      max_length=10						,      return_tensors='np'     )
																			SCREAMING_SNAKE_CASE           =      tokenizer_p.batch_encode_candidates(_A						,      max_length=10						,      return_tensors='np'     )

																			SCREAMING_SNAKE_CASE           =      (2, 2, 10)

																			self.assertEqual(encoded_sentence_r['input_ids'].shape						,      _A     )
																			self.assertEqual(encoded_sentence_r['attention_mask'].shape						,      _A     )
																			self.assertEqual(encoded_sentence_r['token_type_ids'].shape						,      _A     )

																			self.assertEqual(encoded_sentence_p['input_ids'].shape						,      _A     )
																			self.assertEqual(encoded_sentence_p['attention_mask'].shape						,      _A     )
																			self.assertEqual(encoded_sentence_p['token_type_ids'].shape						,      _A     )


'''simple docstring'''





from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
 if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from ...utils.dummy_torch_and_transformers_objects import (
     AudioLDMPipeline,
 )
else:
 from .pipeline_audioldm import AudioLDMPipeline






