






'''simple docstring'''





import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_      = logging.get_logger(__name__)

a_      = {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

a_      = {
    'vocab_file': {
        'microsoft/biogpt': 'https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json',
    },
    'merges_file': {'microsoft/biogpt': 'https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt'},
}

a_      = {
    'microsoft/biogpt': 1_024,
}



def 			__UpperCAmelCase				(lowercase__				)		->				int:

						'''simple docstring'''


						a_				     =   set()
						a_				     =   word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)				)
												a_				     =   char
						return pairs





class 		SCREAMING_SNAKE_CASE__   (     lowercase_    ):
		_UpperCAmelCase         =VOCAB_FILES_NAMES
		_UpperCAmelCase         =PRETRAINED_VOCAB_FILES_MAP
		_UpperCAmelCase         =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_UpperCAmelCase         =['''input_ids''', '''attention_mask''']



		def __init__(    self:    List[Any]				,   a:    Tuple				,   a:    Dict				,   a:    List[Any]="<unk>"				,   a:    Optional[int]="<s>"				,   a:    Any="</s>"				,   a:    List[Any]="</s>"				,   a:    Optional[int]="<pad>"				,   **a:    Union[str, Any]				,   )  ->int:






								'''simple docstring'''


								super().__init__(
								    bos_token=a				,   eos_token=a				,   sep_token=a				,   unk_token=a				,   pad_token=a				,   **a				,   )

								try:
														import sacremoses
								except ImportError:
														raise ImportError(
														    "You need to install sacremoses to use BioGptTokenizer. "
														    "See https://pypi.org/project/sacremoses/ for installation.")

								a_				     =   "en"
								a_				     =   sacremoses
								# cache of sm.MosesTokenizer instance
								a_				     =   {}
								a_				     =   {}
								with open(a				,   encoding="utf-8") as vocab_handle:
														a_				     =   json.load(a)
								a_				     =   {v: k for k, v in self.encoder.items()}
								with open(a				,   encoding="utf-8") as merges_handle:
														a_				     =   merges_handle.read().split("\n")[:-1]
								a_				     =   [tuple(merge.split()[:2]) for merge in merges]
								a_				     =   dict(zip(a				,   range(len(a))))
								a_				     =   {}



		@property
		def _lowerCAmelCase     (    self:    Any)  ->int:






								'''simple docstring'''


								return len(self.encoder)



		def _lowerCAmelCase     (    self:    Tuple)  ->List[Any]:






								'''simple docstring'''


								return dict(self.encoder				,   **self.added_tokens_encoder)



		def _lowerCAmelCase     (    self:    Union[str, Any]				,   a:    List[Any]				,   a:    Tuple)  ->Optional[int]:






								'''simple docstring'''


								if lang not in self.cache_moses_tokenizer:
														a_				     =   self.sm.MosesTokenizer(lang=a)
														a_				     =   moses_tokenizer
								return self.cache_moses_tokenizer[lang].tokenize(
								    a				,   aggressive_dash_splits=a				,   return_str=a				,   escape=a)



		def _lowerCAmelCase     (    self:    List[str]				,   a:    Dict				,   a:    Tuple)  ->Optional[Any]:






								'''simple docstring'''


								if lang not in self.cache_moses_detokenizer:
														a_				     =   self.sm.MosesDetokenizer(lang=a)
														a_				     =   moses_detokenizer
								return self.cache_moses_detokenizer[lang].detokenize(a)



		def _lowerCAmelCase     (    self:    int				,   a:    List[Any])  ->str:






								'''simple docstring'''


								a_				     =   tuple(token[:-1]) + (token[-1] + "</w>",)
								if token in self.cache:
														return self.cache[token]
								a_				     =   get_pairs(a)

								if not pairs:
														return token + "</w>"

								while True:
														a_				     =   min(a				,   key=lambda a: self.bpe_ranks.get(a				,   float("inf")))
														if bigram not in self.bpe_ranks:
																				break
														a_     ,       a_				     =   bigram
														a_				     =   []
														a_				     =   0
														while i < len(a):
																				try:
																										a_				     =   word.index(a				,   a)
																				except ValueError:
																										new_word.extend(word[i:])
																										break
																				else:
																										new_word.extend(word[i:j])
																										a_				     =   j

																				if word[i] == first and i < len(a) - 1 and word[i + 1] == second:
																										new_word.append(first + second)
																										i += 2
																				else:
																										new_word.append(word[i])
																										i += 1
														a_				     =   tuple(a)
														a_				     =   new_word
														if len(a) == 1:
																				break
														else:
																				a_				     =   get_pairs(a)
								a_				     =   " ".join(a)
								if word == "\n  </w>":
														a_				     =   "\n</w>"
								a_				     =   word
								return word



		def _lowerCAmelCase     (    self:    Tuple				,   a:    int				,   a:    Dict=False)  ->Optional[Any]:






								'''simple docstring'''


								if bypass_tokenizer:
														a_				     =   text.split()
								else:
														a_				     =   self.moses_tokenize(a				,   self.lang)

								a_				     =   []
								for token in text:
														if token:
																				split_tokens.extend(list(self.bpe(a).split(" ")))

								return split_tokens



		def _lowerCAmelCase     (    self:    int				,   a:    Optional[Any])  ->Tuple:






								'''simple docstring'''


								return self.encoder.get(a				,   self.encoder.get(self.unk_token))



		def _lowerCAmelCase     (    self:    Any				,   a:    int)  ->Optional[int]:






								'''simple docstring'''


								return self.decoder.get(a				,   self.unk_token)



		def _lowerCAmelCase     (    self:    str				,   a:    Any)  ->Optional[Any]:






								'''simple docstring'''


								a_				     =   [t.replace(" "				,   "").replace("</w>"				,   " ") for t in tokens]
								a_				     =   "".join(a).split()
								# detokenize
								a_				     =   self.moses_detokenize(a				,   self.lang)
								return text



		def _lowerCAmelCase     (    self:    Union[str, Any]				,   a:    List[int]				,   a:    Optional[List[int]] = None)  ->List[int]:






								'''simple docstring'''


								if token_ids_a is None:
														return [self.sep_token_id] + token_ids_a
								a_				     =   [self.sep_token_id]
								return sep + token_ids_a + sep + token_ids_a



		def _lowerCAmelCase     (    self:    Optional[int]				,   a:    List[int]				,   a:    Optional[List[int]] = None				,   a:    bool = False)  ->List[int]:






								'''simple docstring'''


								if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=a				,   token_ids_a=a				,   already_has_special_tokens=a)
								# no bos used in fairseq
								if token_ids_a is not None:
														return [1] + ([0] * len(a)) + [1] + ([0] * len(a))
								return [1] + ([0] * len(a))



		def _lowerCAmelCase     (    self:    Dict				,   a:    List[int]				,   a:    Optional[List[int]] = None)  ->List[int]:






								'''simple docstring'''


								a_				     =   [self.sep_token_id]

								# no bos used in fairseq
								if token_ids_a is None:
														return len(token_ids_a + sep) * [0]
								return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



		def _lowerCAmelCase     (    self:    Union[str, Any]				,   a:    str				,   a:    Optional[str] = None)  ->Tuple[str]:






								'''simple docstring'''


								if not os.path.isdir(a):
														logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
														return
								a_				     =   os.path.join(
								    a				,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
								a_				     =   os.path.join(
								    a				,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"])

								with open(a				,   "w"				,   encoding="utf-8") as f:
														f.write(json.dumps(self.encoder				,   indent=2				,   sort_keys=a				,   ensure_ascii=a) + "\n")

								a_				     =   0
								with open(a				,   "w"				,   encoding="utf-8") as writer:
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,   key=lambda a: kv[1]):
																				if index != token_index:
																										logger.warning(
																										    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																										    " Please check that the tokenizer is not corrupted!")
																										a_				     =   token_index
																				writer.write(" ".join(a) + "\n")
																				index += 1

								return vocab_file, merge_file



		def __getstate__(    self:    Any)  ->List[str]:






								'''simple docstring'''


								a_				     =   self.__dict__.copy()
								a_				     =   None
								return state




		def __setstate__(    self:    Tuple				,   a:    List[str])  ->str:






								'''simple docstring'''


								a_				     =   d

								try:
														import sacremoses
								except ImportError:
														raise ImportError(
														    "You need to install sacremoses to use XLMTokenizer. "
														    "See https://pypi.org/project/sacremoses/ for installation.")

								a_				     =   sacremoses











'''simple docstring'''





import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
							from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

							from .test_feature_extraction_speechta import floats_list


a_      = get_tests_dir('fixtures/test_sentencepiece_bpe_char.model')





@require_torch
class 		SCREAMING_SNAKE_CASE__   (     unittest.TestCase    ):



		def _lowerCAmelCase     (    self:    str)  ->Optional[Any]:






								'''simple docstring'''


								a_				     =   tempfile.mkdtemp()

								a_				     =   SpeechTaTokenizer(a)
								tokenizer.save_pretrained(self.tmpdirname)

								a_				     =   {
								    "feature_size": 1,
								    "padding_value": 0.0,
								    "sampling_rate": 1_60_00,
								    "do_normalize": False,
								    "num_mel_bins": 80,
								    "hop_length": 16,
								    "win_length": 64,
								    "win_function": "hann_window",
								    "fmin": 80,
								    "fmax": 76_00,
								    "mel_floor": 1e-10,
								    "reduction_factor": 2,
								    "return_attention_mask": True,
								}

								a_				     =   os.path.join(self.tmpdirname				,   a)
								with open(self.feature_extraction_file				,   "w"				,   encoding="utf-8") as fp:
														fp.write(json.dumps(a) + "\n")



		def _lowerCAmelCase     (    self:    Any				,   **a:    List[Any])  ->Optional[Any]:






								'''simple docstring'''


								return SpeechTaTokenizer.from_pretrained(self.tmpdirname				,   **a)



		def _lowerCAmelCase     (    self:    Union[str, Any]				,   **a:    int)  ->str:






								'''simple docstring'''


								return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname				,   **a)



		def _lowerCAmelCase     (    self:    int)  ->Optional[Any]:






								'''simple docstring'''


								shutil.rmtree(self.tmpdirname)



		def _lowerCAmelCase     (    self:    Any)  ->str:






								'''simple docstring'''


								a_				     =   self.get_tokenizer()
								a_				     =   self.get_feature_extractor()

								a_				     =   SpeechTaProcessor(tokenizer=a				,   feature_extractor=a)

								processor.save_pretrained(self.tmpdirname)
								a_				     =   SpeechTaProcessor.from_pretrained(self.tmpdirname)

								self.assertEqual(processor.tokenizer.get_vocab()				,   tokenizer.get_vocab())
								self.assertIsInstance(processor.tokenizer				,   a)

								self.assertEqual(processor.feature_extractor.to_json_string()				,   feature_extractor.to_json_string())
								self.assertIsInstance(processor.feature_extractor				,   a)



		def _lowerCAmelCase     (    self:    List[Any])  ->Optional[int]:






								'''simple docstring'''


								a_				     =   SpeechTaProcessor(tokenizer=self.get_tokenizer()				,   feature_extractor=self.get_feature_extractor())
								processor.save_pretrained(self.tmpdirname)

								a_				     =   self.get_tokenizer(bos_token="(BOS)"				,   eos_token="(EOS)")
								a_				     =   self.get_feature_extractor(do_normalize=a				,   padding_value=1.0)

								a_				     =   SpeechTaProcessor.from_pretrained(
								    self.tmpdirname				,   bos_token="(BOS)"				,   eos_token="(EOS)"				,   do_normalize=a				,   padding_value=1.0)

								self.assertEqual(processor.tokenizer.get_vocab()				,   tokenizer_add_kwargs.get_vocab())
								self.assertIsInstance(processor.tokenizer				,   a)

								self.assertEqual(processor.feature_extractor.to_json_string()				,   feature_extractor_add_kwargs.to_json_string())
								self.assertIsInstance(processor.feature_extractor				,   a)



		def _lowerCAmelCase     (    self:    Any)  ->List[Any]:






								'''simple docstring'''


								a_				     =   self.get_feature_extractor()
								a_				     =   self.get_tokenizer()

								a_				     =   SpeechTaProcessor(tokenizer=a				,   feature_extractor=a)

								a_				     =   floats_list((3, 10_00))

								a_				     =   feature_extractor(audio=a				,   return_tensors="np")
								a_				     =   processor(audio=a				,   return_tensors="np")

								for key in input_feat_extract.keys():
														self.assertAlmostEqual(input_feat_extract[key].sum()				,   input_processor[key].sum()				,   delta=1e-2)



		def _lowerCAmelCase     (    self:    Tuple)  ->Union[str, Any]:






								'''simple docstring'''


								a_				     =   self.get_feature_extractor()
								a_				     =   self.get_tokenizer()

								a_				     =   SpeechTaProcessor(tokenizer=a				,   feature_extractor=a)

								a_				     =   floats_list((3, 10_00))

								a_				     =   feature_extractor(audio_target=a				,   return_tensors="np")
								a_				     =   processor(audio_target=a				,   return_tensors="np")

								for key in input_feat_extract.keys():
														self.assertAlmostEqual(input_feat_extract[key].sum()				,   input_processor[key].sum()				,   delta=1e-2)



		def _lowerCAmelCase     (    self:    str)  ->Dict:






								'''simple docstring'''


								a_				     =   self.get_feature_extractor()
								a_				     =   self.get_tokenizer()

								a_				     =   SpeechTaProcessor(tokenizer=a				,   feature_extractor=a)

								a_				     =   "This is a test string"

								a_				     =   processor(text=a)
								a_				     =   tokenizer(a)

								for key in encoded_tok.keys():
														self.assertListEqual(encoded_tok[key]				,   encoded_processor[key])



		def _lowerCAmelCase     (    self:    List[Any])  ->Any:






								'''simple docstring'''


								a_				     =   self.get_feature_extractor()
								a_				     =   self.get_tokenizer()

								a_				     =   SpeechTaProcessor(tokenizer=a				,   feature_extractor=a)

								a_				     =   "This is a test string"

								a_				     =   processor(text_target=a)
								a_				     =   tokenizer(a)

								for key in encoded_tok.keys():
														self.assertListEqual(encoded_tok[key]				,   encoded_processor[key])



		def _lowerCAmelCase     (    self:    List[Any])  ->Any:






								'''simple docstring'''


								a_				     =   self.get_feature_extractor()
								a_				     =   self.get_tokenizer()

								a_				     =   SpeechTaProcessor(tokenizer=a				,   feature_extractor=a)

								a_				     =   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

								a_				     =   processor.batch_decode(a)
								a_				     =   tokenizer.batch_decode(a)

								self.assertListEqual(a				,   a)




		def _lowerCAmelCase     (    self:    Optional[int])  ->Optional[int]:






								'''simple docstring'''


								a_				     =   self.get_feature_extractor()
								a_				     =   self.get_tokenizer()

								a_				     =   SpeechTaProcessor(tokenizer=a				,   feature_extractor=a)

								self.assertListEqual(
								    processor.model_input_names				,   feature_extractor.model_input_names				,   msg="`processor` and `feature_extractor` model input names do not match"				,   )



