




#
# Sort large text files in a minimum amount of memory
#
import argparse
import os





class     snake_case  :


   '''simple docstring'''







   UpperCamelCase__	:       List[Any]     =      "block_{0}.dat"




   def __init__(					self     :	Any						,    lowerCamelCase_     :	str			)					->Any:
     '''simple docstring'''







     UpperCAmelCase__    						=   filename
     UpperCAmelCase__    						=   []




   def 	UpperCAmelCase       (					self     :	Tuple						,    lowerCamelCase_     :	Tuple						,    lowerCamelCase_     :	Any			)					->Optional[Any]:
     '''simple docstring'''







     UpperCAmelCase__    						=   self.BLOCK_FILENAME_FORMAT.format(lowerCamelCase_			)
     with open(lowerCamelCase_						,    """w"""			) as file:
       file.write(lowerCamelCase_			)
     self.block_filenames.append(lowerCamelCase_			)




   def 	UpperCAmelCase       (					self     :	List[str]			)					->Optional[int]:
     '''simple docstring'''







     return self.block_filenames




   def 	UpperCAmelCase       (					self     :	Union[str, Any]						,    lowerCamelCase_     :	List[str]						,    lowerCamelCase_     :	Optional[int]=None			)					->int:
     '''simple docstring'''







     UpperCAmelCase__    						=   0
     with open(self.filename			) as file:
       while True:
         UpperCAmelCase__    						=   file.readlines(lowerCamelCase_			)

         if lines == []:
           break

         if sort_key is None:
           lines.sort()
         else:
           lines.sort(key=lowerCamelCase_			)

         self.write_block("""""".join(lowerCamelCase_			)						,    lowerCamelCase_			)
         i += 1







   def 	UpperCAmelCase       (					self     :	List[str]			)					->Any:
     '''simple docstring'''







     map(os.remove						,    self.block_filenames			)





class     snake_case  :


   '''simple docstring'''







   def 	UpperCAmelCase       (					self     :	Tuple						,    lowerCamelCase_     :	Optional[Any]			)					->List[str]:
     '''simple docstring'''







     UpperCAmelCase__    						=   -1
     UpperCAmelCase__    						=   None

     for i in range(len(lowerCamelCase_			)			):
       if min_str is None or choices[i] < min_str:
         UpperCAmelCase__    						=   i

     return min_index





class     snake_case  :


   '''simple docstring'''







   def __init__(					self     :	str						,    lowerCamelCase_     :	int			)					->str:
     '''simple docstring'''







     UpperCAmelCase__    						=   files
     UpperCAmelCase__    						=   set()
     UpperCAmelCase__    						=   len(lowerCamelCase_			)
     UpperCAmelCase__    						=   {i: None for i in range(self.num_buffers			)}




   def 	UpperCAmelCase       (					self     :	Dict			)					->Optional[int]:
     '''simple docstring'''







     return {
         i: self.buffers[i] for i in range(self.num_buffers			) if i not in self.empty
     }




   def 	UpperCAmelCase       (					self     :	Tuple			)					->Optional[int]:
     '''simple docstring'''







     for i in range(self.num_buffers			):
       if self.buffers[i] is None and i not in self.empty:
         UpperCAmelCase__    						=   self.files[i].readline()

         if self.buffers[i] == "":
           self.empty.add(lowerCamelCase_			)
           self.files[i].close()

     if len(self.empty			) == self.num_buffers:
       return False

     return True







   def 	UpperCAmelCase       (					self     :	Union[str, Any]						,    lowerCamelCase_     :	Any			)					->str:
     '''simple docstring'''







     UpperCAmelCase__    						=   self.buffers[index]
     UpperCAmelCase__    						=   None

     return value





class     snake_case  :


   '''simple docstring'''







   def __init__(					self     :	List[str]						,    lowerCamelCase_     :	Dict			)					->Any:
     '''simple docstring'''







     UpperCAmelCase__    						=   merge_strategy




   def 	UpperCAmelCase       (					self     :	List[Any]						,    lowerCamelCase_     :	List[Any]						,    lowerCamelCase_     :	Optional[int]						,    lowerCamelCase_     :	Optional[Any]			)					->Any:
     '''simple docstring'''







     UpperCAmelCase__    						=   FilesArray(self.get_file_handles(lowerCamelCase_						,    lowerCamelCase_			)			)
     with open(lowerCamelCase_						,    """w"""						,    lowerCamelCase_			) as outfile:
       while buffers.refresh():
         UpperCAmelCase__    						=   self.merge_strategy.select(buffers.get_dict()			)
         outfile.write(buffers.unshift(lowerCamelCase_			)			)







   def 	UpperCAmelCase       (					self     :	List[Any]						,    lowerCamelCase_     :	Union[str, Any]						,    lowerCamelCase_     :	Union[str, Any]			)					->str:
     '''simple docstring'''







     UpperCAmelCase__    						=   {}

     for i in range(len(lowerCamelCase_			)			):
       UpperCAmelCase__    						=   open(filenames[i]						,    """r"""						,    lowerCamelCase_			)  # noqa: UP015

     return files





class     snake_case  :


   '''simple docstring'''







   def __init__(					self     :	List[str]						,    lowerCamelCase_     :	Union[str, Any]			)					->Tuple:
     '''simple docstring'''







     UpperCAmelCase__    						=   block_size




   def 	UpperCAmelCase       (					self     :	Optional[Any]						,    lowerCamelCase_     :	Tuple						,    lowerCamelCase_     :	Dict=None			)					->List[str]:
     '''simple docstring'''







     UpperCAmelCase__    						=   self.get_number_blocks(lowerCamelCase_						,    self.block_size			)
     UpperCAmelCase__    						=   FileSplitter(lowerCamelCase_			)
     splitter.split(self.block_size						,    lowerCamelCase_			)

     UpperCAmelCase__    						=   FileMerger(NWayMerge()			)
     UpperCAmelCase__    						=   self.block_size / (num_blocks + 1)
     merger.merge(splitter.get_block_filenames()						,    filename + """.out"""						,    lowerCamelCase_			)

     splitter.cleanup()







   def 	UpperCAmelCase       (					self     :	Tuple						,    lowerCamelCase_     :	List[str]						,    lowerCamelCase_     :	Optional[Any]			)					->str:
     '''simple docstring'''







     return (os.stat(lowerCamelCase_			).st_size / block_size) + 1



def 						lowercase		(     _lowerCAmelCase      ):
  if string[-1].lower() == "k":
    return int(string[:-1]      ) * 1024
  elif string[-1].lower() == "m":
    return int(string[:-1]      ) * 1024 * 1024
  elif string[-1].lower() == "g":
    return int(string[:-1]      ) * 1024 * 1024 * 1024
  else:
    return int(_lowerCAmelCase      )


def 						lowercase		(     ):
  UpperCAmelCase__    						=   argparse.ArgumentParser()
  parser.add_argument(
      """-m"""   ,     """--mem"""   ,     help="""amount of memory to use for sorting"""   ,     default="""100M"""      )
  parser.add_argument(
      """filename"""   ,     metavar="""<filename>"""   ,     nargs=1   ,     help="""name of file to sort"""      )
  UpperCAmelCase__    						=   parser.parse_args()

  UpperCAmelCase__    						=   ExternalSort(parse_memory(args.mem      )      )
  sorter.sort(args.filename[0]      )


if __name__ == "__main__":
    main()










import shutil
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SpeechaTextTokenizer, is_speech_available
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_speech_to_text import floats_list


if is_speech_available():
   from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor


snake_case__ :						Union[str, Any]     =			get_tests_dir('''fixtures/test_sentencepiece.model''')





@require_torch
@require_torchaudio
@require_sentencepiece
class     snake_case  (	unittest.TestCase							):


   '''simple docstring'''







   def 	UpperCAmelCase       (					self     :	str			)					->str:
     '''simple docstring'''







     UpperCAmelCase__    						=   tempfile.mkdtemp()

     UpperCAmelCase__    						=   ["""<s>""", """<pad>""", """</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est"""]
     UpperCAmelCase__    						=   dict(zip(lowerCamelCase_						,    range(len(lowerCamelCase_			)			)			)			)
     UpperCAmelCase__    						=   Path(self.tmpdirname			)
     save_json(lowerCamelCase_						,    save_dir / VOCAB_FILES_NAMES["""vocab_file"""]			)
     if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
       copyfile(lowerCamelCase_						,    save_dir / VOCAB_FILES_NAMES["""spm_file"""]			)

     UpperCAmelCase__    						=   SpeechaTextTokenizer.from_pretrained(self.tmpdirname			)
     tokenizer.save_pretrained(self.tmpdirname			)

     UpperCAmelCase__    						=   {
         """feature_size""": 24,
         """num_mel_bins""": 24,
         """padding_value""": 0.0,
         """sampling_rate""": 1_6000,
         """return_attention_mask""": False,
         """do_normalize""": True,
     }
     save_json(lowerCamelCase_						,    save_dir / FEATURE_EXTRACTOR_NAME			)




   def 	UpperCAmelCase       (					self     :	Union[str, Any]						,    **lowerCamelCase_     :	Tuple			)					->Any:
     '''simple docstring'''







     return SpeechaTextTokenizer.from_pretrained(self.tmpdirname						,    **lowerCamelCase_			)




   def 	UpperCAmelCase       (					self     :	Dict						,    **lowerCamelCase_     :	str			)					->Optional[Any]:
     '''simple docstring'''







     return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname						,    **lowerCamelCase_			)




   def 	UpperCAmelCase       (					self     :	List[str]			)					->Dict:
     '''simple docstring'''







     shutil.rmtree(self.tmpdirname			)




   def 	UpperCAmelCase       (					self     :	str			)					->Tuple:
     '''simple docstring'''







     UpperCAmelCase__    						=   self.get_tokenizer()
     UpperCAmelCase__    						=   self.get_feature_extractor()

     UpperCAmelCase__    						=   SpeechaTextProcessor(tokenizer=lowerCamelCase_						,    feature_extractor=lowerCamelCase_			)

     processor.save_pretrained(self.tmpdirname			)
     UpperCAmelCase__    						=   SpeechaTextProcessor.from_pretrained(self.tmpdirname			)

     self.assertEqual(processor.tokenizer.get_vocab()						,    tokenizer.get_vocab()			)
     self.assertIsInstance(processor.tokenizer						,    lowerCamelCase_			)

     self.assertEqual(processor.feature_extractor.to_json_string()						,    feature_extractor.to_json_string()			)
     self.assertIsInstance(processor.feature_extractor						,    lowerCamelCase_			)




   def 	UpperCAmelCase       (					self     :	List[str]			)					->Tuple:
     '''simple docstring'''







     UpperCAmelCase__    						=   SpeechaTextProcessor(
         tokenizer=self.get_tokenizer()						,    feature_extractor=self.get_feature_extractor()			)
     processor.save_pretrained(self.tmpdirname			)

     UpperCAmelCase__    						=   self.get_tokenizer(bos_token="""(BOS)"""						,    eos_token="""(EOS)"""			)
     UpperCAmelCase__    						=   self.get_feature_extractor(do_normalize=lowerCamelCase_						,    padding_value=1.0			)

     UpperCAmelCase__    						=   SpeechaTextProcessor.from_pretrained(
         self.tmpdirname						,    bos_token="""(BOS)"""						,    eos_token="""(EOS)"""						,    do_normalize=lowerCamelCase_						,    padding_value=1.0			)

     self.assertEqual(processor.tokenizer.get_vocab()						,    tokenizer_add_kwargs.get_vocab()			)
     self.assertIsInstance(processor.tokenizer						,    lowerCamelCase_			)

     self.assertEqual(processor.feature_extractor.to_json_string()						,    feature_extractor_add_kwargs.to_json_string()			)
     self.assertIsInstance(processor.feature_extractor						,    lowerCamelCase_			)




   def 	UpperCAmelCase       (					self     :	Union[str, Any]			)					->Tuple:
     '''simple docstring'''







     UpperCAmelCase__    						=   self.get_feature_extractor()
     UpperCAmelCase__    						=   self.get_tokenizer()

     UpperCAmelCase__    						=   SpeechaTextProcessor(tokenizer=lowerCamelCase_						,    feature_extractor=lowerCamelCase_			)

     UpperCAmelCase__    						=   floats_list((3, 1000)			)

     UpperCAmelCase__    						=   feature_extractor(lowerCamelCase_						,    return_tensors="""np"""			)
     UpperCAmelCase__    						=   processor(lowerCamelCase_						,    return_tensors="""np"""			)

     for key in input_feat_extract.keys():
       self.assertAlmostEqual(input_feat_extract[key].sum()						,    input_processor[key].sum()						,    delta=1E-2			)




   def 	UpperCAmelCase       (					self     :	str			)					->Union[str, Any]:
     '''simple docstring'''







     UpperCAmelCase__    						=   self.get_feature_extractor()
     UpperCAmelCase__    						=   self.get_tokenizer()

     UpperCAmelCase__    						=   SpeechaTextProcessor(tokenizer=lowerCamelCase_						,    feature_extractor=lowerCamelCase_			)

     UpperCAmelCase__    						=   """This is a test string"""

     UpperCAmelCase__    						=   processor(text=lowerCamelCase_			)

     UpperCAmelCase__    						=   tokenizer(lowerCamelCase_			)

     for key in encoded_tok.keys():
       self.assertListEqual(encoded_tok[key]						,    encoded_processor[key]			)




   def 	UpperCAmelCase       (					self     :	Optional[Any]			)					->Tuple:
     '''simple docstring'''







     UpperCAmelCase__    						=   self.get_feature_extractor()
     UpperCAmelCase__    						=   self.get_tokenizer()

     UpperCAmelCase__    						=   SpeechaTextProcessor(tokenizer=lowerCamelCase_						,    feature_extractor=lowerCamelCase_			)

     UpperCAmelCase__    						=   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

     UpperCAmelCase__    						=   processor.batch_decode(lowerCamelCase_			)
     UpperCAmelCase__    						=   tokenizer.batch_decode(lowerCamelCase_			)

     self.assertListEqual(lowerCamelCase_						,    lowerCamelCase_			)







   def 	UpperCAmelCase       (					self     :	Dict			)					->Tuple:
     '''simple docstring'''







     UpperCAmelCase__    						=   self.get_feature_extractor()
     UpperCAmelCase__    						=   self.get_tokenizer()

     UpperCAmelCase__    						=   SpeechaTextProcessor(tokenizer=lowerCamelCase_						,    feature_extractor=lowerCamelCase_			)

     self.assertListEqual(
         processor.model_input_names						,    feature_extractor.model_input_names						,    msg="""`processor` and `feature_extractor` model input names do not match"""						,    )




