





'''simple docstring'''







import shutil
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SpeechaTextTokenizer, is_speech_available
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_speech_to_text import floats_list


if is_speech_available():
					from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor


snake_case_  :							Optional[Any]            =	get_tests_dir("fixtures/test_sentencepiece.model")


@require_torch
@require_torchaudio
@require_sentencepiece
class        __a   (unittest.TestCase  ):

							def      UpperCAmelCase__						(				self				:  List[str] )       ->							Optional[int]:







													"""simple docstring"""
													UpperCAmelCase_      :   int      =	tempfile.mkdtemp()

													UpperCAmelCase_      :   Union[str, Any]      =	['''<s>''', '''<pad>''', '''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']
													UpperCAmelCase_      :   Optional[Any]      =	dict(zip(__magic_name__							,  range(len(__magic_name__ ) ) ) )
													UpperCAmelCase_      :   Optional[int]      =	Path(self.tmpdirname )
													save_json(__magic_name__							,  save_dir / VOCAB_FILES_NAMES['''vocab_file'''] )
													if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																			copyfile(__magic_name__							,  save_dir / VOCAB_FILES_NAMES['''spm_file'''] )

													UpperCAmelCase_      :   Dict      =	SpeechaTextTokenizer.from_pretrained(self.tmpdirname )
													tokenizer.save_pretrained(self.tmpdirname )

													UpperCAmelCase_      :   Optional[int]      =	{
													    '''feature_size''': 24,
													    '''num_mel_bins''': 24,
													    '''padding_value''': 0.0,
													    '''sampling_rate''': 1_60_00,
													    '''return_attention_mask''': False,
													    '''do_normalize''': True,
													}
													save_json(__magic_name__							,  save_dir / FEATURE_EXTRACTOR_NAME )

							def      UpperCAmelCase__						(				self				:  Optional[Any]							,  **__magic_name__				:  Tuple )       ->							str:







													"""simple docstring"""
													return SpeechaTextTokenizer.from_pretrained(self.tmpdirname							,  **__magic_name__ )

							def      UpperCAmelCase__						(				self				:  str							,  **__magic_name__				:  str )       ->							Optional[int]:







													"""simple docstring"""
													return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname							,  **__magic_name__ )

							def      UpperCAmelCase__						(				self				:  Optional[Any] )       ->							Optional[int]:







													"""simple docstring"""
													shutil.rmtree(self.tmpdirname )

							def      UpperCAmelCase__						(				self				:  Optional[Any] )       ->							List[Any]:







													"""simple docstring"""
													UpperCAmelCase_      :   List[str]      =	self.get_tokenizer()
													UpperCAmelCase_      :   Any      =	self.get_feature_extractor()

													UpperCAmelCase_      :   Optional[Any]      =	SpeechaTextProcessor(tokenizer=__magic_name__							,  feature_extractor=__magic_name__ )

													processor.save_pretrained(self.tmpdirname )
													UpperCAmelCase_      :   Tuple      =	SpeechaTextProcessor.from_pretrained(self.tmpdirname )

													self.assertEqual(processor.tokenizer.get_vocab()							,  tokenizer.get_vocab() )
													self.assertIsInstance(processor.tokenizer							,  __magic_name__ )

													self.assertEqual(processor.feature_extractor.to_json_string()							,  feature_extractor.to_json_string() )
													self.assertIsInstance(processor.feature_extractor							,  __magic_name__ )

							def      UpperCAmelCase__						(				self				:  int )       ->							Union[str, Any]:







													"""simple docstring"""
													UpperCAmelCase_      :   Optional[int]      =	SpeechaTextProcessor(
													    tokenizer=self.get_tokenizer()							,  feature_extractor=self.get_feature_extractor() )
													processor.save_pretrained(self.tmpdirname )

													UpperCAmelCase_      :   List[Any]      =	self.get_tokenizer(bos_token='''(BOS)'''							,  eos_token='''(EOS)''' )
													UpperCAmelCase_      :   Any      =	self.get_feature_extractor(do_normalize=__magic_name__							,  padding_value=1.0 )

													UpperCAmelCase_      :   Union[str, Any]      =	SpeechaTextProcessor.from_pretrained(
													    self.tmpdirname							,  bos_token='''(BOS)'''							,  eos_token='''(EOS)'''							,  do_normalize=__magic_name__							,  padding_value=1.0 )

													self.assertEqual(processor.tokenizer.get_vocab()							,  tokenizer_add_kwargs.get_vocab() )
													self.assertIsInstance(processor.tokenizer							,  __magic_name__ )

													self.assertEqual(processor.feature_extractor.to_json_string()							,  feature_extractor_add_kwargs.to_json_string() )
													self.assertIsInstance(processor.feature_extractor							,  __magic_name__ )

							def      UpperCAmelCase__						(				self				:  Optional[int] )       ->							Union[str, Any]:







													"""simple docstring"""
													UpperCAmelCase_      :   str      =	self.get_feature_extractor()
													UpperCAmelCase_      :   Optional[int]      =	self.get_tokenizer()

													UpperCAmelCase_      :   Union[str, Any]      =	SpeechaTextProcessor(tokenizer=__magic_name__							,  feature_extractor=__magic_name__ )

													UpperCAmelCase_      :   Tuple      =	floats_list((3, 10_00) )

													UpperCAmelCase_      :   Optional[Any]      =	feature_extractor(__magic_name__							,  return_tensors='''np''' )
													UpperCAmelCase_      :   List[str]      =	processor(__magic_name__							,  return_tensors='''np''' )

													for key in input_feat_extract.keys():
																			self.assertAlmostEqual(input_feat_extract[key].sum()							,  input_processor[key].sum()							,  delta=1E-2 )

							def      UpperCAmelCase__						(				self				:  str )       ->							Optional[int]:







													"""simple docstring"""
													UpperCAmelCase_      :   Dict      =	self.get_feature_extractor()
													UpperCAmelCase_      :   List[str]      =	self.get_tokenizer()

													UpperCAmelCase_      :   Optional[int]      =	SpeechaTextProcessor(tokenizer=__magic_name__							,  feature_extractor=__magic_name__ )

													UpperCAmelCase_      :   Optional[Any]      =	'''This is a test string'''

													UpperCAmelCase_      :   List[Any]      =	processor(text=__magic_name__ )

													UpperCAmelCase_      :   int      =	tokenizer(__magic_name__ )

													for key in encoded_tok.keys():
																			self.assertListEqual(encoded_tok[key]							,  encoded_processor[key] )

							def      UpperCAmelCase__						(				self				:  str )       ->							List[Any]:







													"""simple docstring"""
													UpperCAmelCase_      :   int      =	self.get_feature_extractor()
													UpperCAmelCase_      :   Optional[Any]      =	self.get_tokenizer()

													UpperCAmelCase_      :   str      =	SpeechaTextProcessor(tokenizer=__magic_name__							,  feature_extractor=__magic_name__ )

													UpperCAmelCase_      :   Union[str, Any]      =	[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

													UpperCAmelCase_      :   Any      =	processor.batch_decode(__magic_name__ )
													UpperCAmelCase_      :   List[Any]      =	tokenizer.batch_decode(__magic_name__ )

													self.assertListEqual(__magic_name__							,  __magic_name__ )



							def      UpperCAmelCase__						(				self				:  Optional[int] )       ->							Any:







													"""simple docstring"""
													UpperCAmelCase_      :   Tuple      =	self.get_feature_extractor()
													UpperCAmelCase_      :   str      =	self.get_tokenizer()

													UpperCAmelCase_      :   List[Any]      =	SpeechaTextProcessor(tokenizer=__magic_name__							,  feature_extractor=__magic_name__ )

													self.assertListEqual(
													    processor.model_input_names							,  feature_extractor.model_input_names							,  msg='''`processor` and `feature_extractor` model input names do not match'''							,  )













'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_A      :     Optional[Any]       	={
    '''configuration_realm''': ['''REALM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''RealmConfig'''],
    '''tokenization_realm''': ['''RealmTokenizer'''],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A      :     str       	=['''RealmTokenizerFast''']

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A      :     List[Any]       	=[
			    '''REALM_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''RealmEmbedder''',
			    '''RealmForOpenQA''',
			    '''RealmKnowledgeAugEncoder''',
			    '''RealmPreTrainedModel''',
			    '''RealmReader''',
			    '''RealmScorer''',
			    '''load_tf_weights_in_realm''',
			]
			_A      :     Union[str, Any]       	=['''RealmRetriever''']


if TYPE_CHECKING:
			from .configuration_realm import REALM_PRETRAINED_CONFIG_ARCHIVE_MAP, RealmConfig
			from .tokenization_realm import RealmTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_realm import RealmTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_realm import (
						    REALM_PRETRAINED_MODEL_ARCHIVE_LIST,
						    RealmEmbedder,
						    RealmForOpenQA,
						    RealmKnowledgeAugEncoder,
						    RealmPreTrainedModel,
						    RealmReader,
						    RealmScorer,
						    load_tf_weights_in_realm,
						)
						from .retrieval_realm import RealmRetriever


else:
			import sys

			_A      :     int       	=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

