



'''simple docstring'''






import shutil
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SpeechaTextTokenizer, is_speech_available
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_speech_to_text import floats_list


if is_speech_available():
						from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor


lowerCamelCase__	  =		get_tests_dir('fixtures/test_sentencepiece.model')

@require_torch
@require_torchaudio
@require_sentencepiece
class 	_lowerCAmelCase     (    unittest.TestCase       ):






		'''simple docstring'''
		def 						__lowercase      (						self							:       List[Any]		)      ->			str:







									'''simple docstring'''





									_lowercase							:		List[str]				=   tempfile.mkdtemp()

									_lowercase							:		Optional[Any]				=   ['''<s>''', '''<pad>''', '''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']
									_lowercase							:		Optional[Any]				=   dict(zip(UpperCamelCase_      ,		range(len(UpperCamelCase_		)		)		)		)
									_lowercase							:		List[Any]				=   Path(self.tmpdirname		)
									save_json(UpperCamelCase_      ,		save_dir / VOCAB_FILES_NAMES['''vocab_file''']		)
									if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																copyfile(UpperCamelCase_      ,		save_dir / VOCAB_FILES_NAMES['''spm_file''']		)

									_lowercase							:		Any				=   SpeechaTextTokenizer.from_pretrained(self.tmpdirname		)
									tokenizer.save_pretrained(self.tmpdirname		)

									_lowercase							:		List[Any]				=   {
									    '''feature_size''': 24,
									    '''num_mel_bins''': 24,
									    '''padding_value''': 0.0,
									    '''sampling_rate''': 16_000,
									    '''return_attention_mask''': False,
									    '''do_normalize''': True,
									}
									save_json(UpperCamelCase_      ,		save_dir / FEATURE_EXTRACTOR_NAME		)
		def 						__lowercase      (						self							:       Any      ,		**UpperCamelCase_							:       str		)      ->			Optional[int]:







									'''simple docstring'''





									return SpeechaTextTokenizer.from_pretrained(self.tmpdirname      ,		**UpperCamelCase_		)
		def 						__lowercase      (						self							:       Union[str, Any]      ,		**UpperCamelCase_							:       Dict		)      ->			Any:







									'''simple docstring'''





									return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname      ,		**UpperCamelCase_		)
		def 						__lowercase      (						self							:       Optional[Any]		)      ->			int:







									'''simple docstring'''





									shutil.rmtree(self.tmpdirname		)
		def 						__lowercase      (						self							:       int		)      ->			Optional[Any]:







									'''simple docstring'''





									_lowercase							:		Optional[int]				=   self.get_tokenizer()
									_lowercase							:		int				=   self.get_feature_extractor()

									_lowercase							:		str				=   SpeechaTextProcessor(tokenizer=UpperCamelCase_      ,		feature_extractor=UpperCamelCase_		)

									processor.save_pretrained(self.tmpdirname		)
									_lowercase							:		List[Any]				=   SpeechaTextProcessor.from_pretrained(self.tmpdirname		)

									self.assertEqual(processor.tokenizer.get_vocab()      ,		tokenizer.get_vocab()		)
									self.assertIsInstance(processor.tokenizer      ,		UpperCamelCase_		)

									self.assertEqual(processor.feature_extractor.to_json_string()      ,		feature_extractor.to_json_string()		)
									self.assertIsInstance(processor.feature_extractor      ,		UpperCamelCase_		)
		def 						__lowercase      (						self							:       List[Any]		)      ->			Dict:







									'''simple docstring'''





									_lowercase							:		Optional[Any]				=   SpeechaTextProcessor(
									    tokenizer=self.get_tokenizer()      ,		feature_extractor=self.get_feature_extractor()		)
									processor.save_pretrained(self.tmpdirname		)

									_lowercase							:		int				=   self.get_tokenizer(bos_token='''(BOS)'''      ,		eos_token='''(EOS)'''		)
									_lowercase							:		List[Any]				=   self.get_feature_extractor(do_normalize=UpperCamelCase_      ,		padding_value=1.0		)

									_lowercase							:		Union[str, Any]				=   SpeechaTextProcessor.from_pretrained(
									    self.tmpdirname      ,		bos_token='''(BOS)'''      ,		eos_token='''(EOS)'''      ,		do_normalize=UpperCamelCase_      ,		padding_value=1.0		)

									self.assertEqual(processor.tokenizer.get_vocab()      ,		tokenizer_add_kwargs.get_vocab()		)
									self.assertIsInstance(processor.tokenizer      ,		UpperCamelCase_		)

									self.assertEqual(processor.feature_extractor.to_json_string()      ,		feature_extractor_add_kwargs.to_json_string()		)
									self.assertIsInstance(processor.feature_extractor      ,		UpperCamelCase_		)
		def 						__lowercase      (						self							:       str		)      ->			Any:







									'''simple docstring'''





									_lowercase							:		Tuple				=   self.get_feature_extractor()
									_lowercase							:		Union[str, Any]				=   self.get_tokenizer()

									_lowercase							:		Optional[int]				=   SpeechaTextProcessor(tokenizer=UpperCamelCase_      ,		feature_extractor=UpperCamelCase_		)

									_lowercase							:		str				=   floats_list((3, 1_000)		)

									_lowercase							:		Optional[int]				=   feature_extractor(UpperCamelCase_      ,		return_tensors='''np'''		)
									_lowercase							:		Any				=   processor(UpperCamelCase_      ,		return_tensors='''np'''		)

									for key in input_feat_extract.keys():
																self.assertAlmostEqual(input_feat_extract[key].sum()      ,		input_processor[key].sum()      ,		delta=1e-2		)
		def 						__lowercase      (						self							:       List[Any]		)      ->			Dict:







									'''simple docstring'''





									_lowercase							:		Tuple				=   self.get_feature_extractor()
									_lowercase							:		Dict				=   self.get_tokenizer()

									_lowercase							:		List[Any]				=   SpeechaTextProcessor(tokenizer=UpperCamelCase_      ,		feature_extractor=UpperCamelCase_		)

									_lowercase							:		Optional[Any]				=   '''This is a test string'''

									_lowercase							:		Optional[int]				=   processor(text=UpperCamelCase_		)

									_lowercase							:		int				=   tokenizer(UpperCamelCase_		)

									for key in encoded_tok.keys():
																self.assertListEqual(encoded_tok[key]      ,		encoded_processor[key]		)
		def 						__lowercase      (						self							:       Any		)      ->			Dict:







									'''simple docstring'''





									_lowercase							:		Optional[int]				=   self.get_feature_extractor()
									_lowercase							:		Dict				=   self.get_tokenizer()

									_lowercase							:		Optional[int]				=   SpeechaTextProcessor(tokenizer=UpperCamelCase_      ,		feature_extractor=UpperCamelCase_		)

									_lowercase							:		Optional[int]				=   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

									_lowercase							:		Tuple				=   processor.batch_decode(UpperCamelCase_		)
									_lowercase							:		List[Any]				=   tokenizer.batch_decode(UpperCamelCase_		)

									self.assertListEqual(UpperCamelCase_      ,		UpperCamelCase_		)





		def 						__lowercase      (						self							:       List[Any]		)      ->			Tuple:







									'''simple docstring'''





									_lowercase							:		int				=   self.get_feature_extractor()
									_lowercase							:		Tuple				=   self.get_tokenizer()

									_lowercase							:		Union[str, Any]				=   SpeechaTextProcessor(tokenizer=UpperCamelCase_      ,		feature_extractor=UpperCamelCase_		)

									self.assertListEqual(
									    processor.model_input_names      ,		feature_extractor.model_input_names      ,		msg='''`processor` and `feature_extractor` model input names do not match'''      ,		)








'''simple docstring'''






import warnings

from .generation import FlaxGenerationMixin

class 	_lowerCAmelCase     (    __A       ):






		'''simple docstring'''
		# warning at import time
		warnings.warn(
		    'Importing `FlaxGenerationMixin` from `src/transformers/generation_flax_utils.py` is deprecated and will '
		    'be removed in Transformers v5. Import as `from transformers import FlaxGenerationMixin` instead.'     ,		__A     ,		)



