



'''simple docstring'''
#
# Sort large text files in a minimum amount of memory
#
import argparse
import os



class _A	:
		_SCREAMING_SNAKE_CASE       :     Union[str, Any]       =      "block_{0}.dat"





		def __init__(  self			,							__UpperCAmelCase					)							->  Optional[Any]:

					'''simple docstring'''
					__UpperCAmelCase :   int    					=  filename
					__UpperCAmelCase :   int    					=  []





		def 				__A							(  self			,							__UpperCAmelCase			,							__UpperCAmelCase					)							->  Optional[Any]:

					'''simple docstring'''
					__UpperCAmelCase :   Dict    					=  self.BLOCK_FILENAME_FORMAT.format(__UpperCAmelCase					)
					with open(__UpperCAmelCase			,							"""w"""					) as file:
								file.write(__UpperCAmelCase					)
					self.block_filenames.append(__UpperCAmelCase					)





		def 				__A							(  self					)							->  str:

					'''simple docstring'''
					return self.block_filenames





		def 				__A							(  self			,							__UpperCAmelCase			,							__UpperCAmelCase=None					)							->  List[Any]:

					'''simple docstring'''
					__UpperCAmelCase :   Any    					=  0
					with open(self.filename					) as file:
								while True:
											__UpperCAmelCase :   Optional[Any]    					=  file.readlines(__UpperCAmelCase					)

											if lines == []:
														break

											if sort_key is None:
														lines.sort()
											else:
														lines.sort(key=__UpperCAmelCase					)

											self.write_block("""""".join(__UpperCAmelCase					)			,							__UpperCAmelCase					)
											i += 1





		def 				__A							(  self					)							->  Optional[Any]:

					'''simple docstring'''
					map(os.remove			,							self.block_filenames					)



class _A	:





		def 				__A							(  self			,							__UpperCAmelCase					)							->  Any:

					'''simple docstring'''
					__UpperCAmelCase :   Dict    					=  -1
					__UpperCAmelCase :   List[str]    					=  None

					for i in range(len(__UpperCAmelCase					)					):
								if min_str is None or choices[i] < min_str:
											__UpperCAmelCase :   int    					=  i

					return min_index



class _A	:





		def __init__(  self			,							__UpperCAmelCase					)							->  str:

					'''simple docstring'''
					__UpperCAmelCase :   int    					=  files
					__UpperCAmelCase :   List[str]    					=  set()
					__UpperCAmelCase :   Any    					=  len(__UpperCAmelCase					)
					__UpperCAmelCase :   Optional[int]    					=  {i: None for i in range(self.num_buffers					)}





		def 				__A							(  self					)							->  Optional[int]:

					'''simple docstring'''
					return {
					    i: self.buffers[i] for i in range(self.num_buffers					) if i not in self.empty
					}





		def 				__A							(  self					)							->  str:

					'''simple docstring'''
					for i in range(self.num_buffers					):
								if self.buffers[i] is None and i not in self.empty:
											__UpperCAmelCase :   Any    					=  self.files[i].readline()

											if self.buffers[i] == "":
														self.empty.add(__UpperCAmelCase					)
														self.files[i].close()

					if len(self.empty					) == self.num_buffers:
								return False

					return True





		def 				__A							(  self			,							__UpperCAmelCase					)							->  Optional[int]:

					'''simple docstring'''
					__UpperCAmelCase :   Any    					=  self.buffers[index]
					__UpperCAmelCase :   Any    					=  None

					return value



class _A	:





		def __init__(  self			,							__UpperCAmelCase					)							->  Optional[Any]:

					'''simple docstring'''
					__UpperCAmelCase :   Tuple    					=  merge_strategy





		def 				__A							(  self			,							__UpperCAmelCase			,							__UpperCAmelCase			,							__UpperCAmelCase					)							->  Union[str, Any]:

					'''simple docstring'''
					__UpperCAmelCase :   Tuple    					=  FilesArray(self.get_file_handles(__UpperCAmelCase			,							__UpperCAmelCase					)					)
					with open(__UpperCAmelCase			,							"""w"""			,							__UpperCAmelCase					) as outfile:
								while buffers.refresh():
											__UpperCAmelCase :   List[Any]    					=  self.merge_strategy.select(buffers.get_dict()					)
											outfile.write(buffers.unshift(__UpperCAmelCase					)					)





		def 				__A							(  self			,							__UpperCAmelCase			,							__UpperCAmelCase					)							->  List[str]:

					'''simple docstring'''
					__UpperCAmelCase :   Tuple    					=  {}

					for i in range(len(__UpperCAmelCase					)					):
								__UpperCAmelCase :   Any    					=  open(filenames[i]			,							"""r"""			,							__UpperCAmelCase					)  # noqa: UP015

					return files



class _A	:





		def __init__(  self			,							__UpperCAmelCase					)							->  Any:

					'''simple docstring'''
					__UpperCAmelCase :   int    					=  block_size





		def 				__A							(  self			,							__UpperCAmelCase			,							__UpperCAmelCase=None					)							->  Tuple:

					'''simple docstring'''
					__UpperCAmelCase :   List[str]    					=  self.get_number_blocks(__UpperCAmelCase			,							self.block_size					)
					__UpperCAmelCase :   List[Any]    					=  FileSplitter(__UpperCAmelCase					)
					splitter.split(self.block_size			,							__UpperCAmelCase					)

					__UpperCAmelCase :   List[Any]    					=  FileMerger(NWayMerge()					)
					__UpperCAmelCase :   str    					=  self.block_size / (num_blocks + 1)
					merger.merge(splitter.get_block_filenames()			,							filename + """.out"""			,							__UpperCAmelCase					)

					splitter.cleanup()





		def 				__A							(  self			,							__UpperCAmelCase			,							__UpperCAmelCase					)							->  Tuple:

					'''simple docstring'''
					return (os.stat(__UpperCAmelCase					).st_size / block_size) + 1


def 	lowercase_						( lowerCAmelCase__						:	str			):

			"""simple docstring"""





			if string[-1].lower() == "k":
						return int(string[:-1]			) * 1024
			elif string[-1].lower() == "m":
						return int(string[:-1]			) * 1024 * 1024
			elif string[-1].lower() == "g":
						return int(string[:-1]			) * 1024 * 1024 * 1024
			else:
						return int(lowerCAmelCase__			)


def 	lowercase_						( ):

			"""simple docstring"""





			__UpperCAmelCase :   Optional[Any]    					=  argparse.ArgumentParser()
			parser.add_argument(
			    """-m"""      ,				"""--mem"""      ,				help="""amount of memory to use for sorting"""      ,				default="""100M"""			)
			parser.add_argument(
			    """filename"""      ,				metavar="""<filename>"""      ,				nargs=1      ,				help="""name of file to sort"""			)
			__UpperCAmelCase :   Tuple    					=  parser.parse_args()

			__UpperCAmelCase :   Dict    					=  ExternalSort(parse_memory(args.mem			)			)
			sorter.sort(args.filename[0]			)


if __name__ == "__main__":
					main()









'''simple docstring'''
import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


_UpperCamelCase									=      logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
_UpperCamelCase									=      direct_transformers_import(Path(__file__).parent)


_UpperCamelCase									=      {
    '''AutoTokenizer''': '''PreTrainedTokenizerBase''',
    '''AutoFeatureExtractor''': '''FeatureExtractionMixin''',
    '''AutoImageProcessor''': '''ImageProcessingMixin''',
}



class _A	(     __SCREAMING_SNAKE_CASE     ):
		_SCREAMING_SNAKE_CASE       :     str       =      ["feature_extractor", "tokenizer"]
		# Names need to be attr_class for attr in attributes
		_SCREAMING_SNAKE_CASE       :     Optional[int]       =      None
		_SCREAMING_SNAKE_CASE       :     str       =      None
		_SCREAMING_SNAKE_CASE       :     List[str]       =      None





		def __init__(  self			,							*__UpperCAmelCase			,							**__UpperCAmelCase					)							->  Optional[int]:

					'''simple docstring'''
					# Sanitize args and kwargs
					for key in kwargs:
								if key not in self.attributes:
											raise TypeError(f'Unexpected keyword argument {key}.'					)
					for arg, attribute_name in zip(__UpperCAmelCase			,							self.attributes					):
								if attribute_name in kwargs:
											raise TypeError(f'Got multiple values for argument {attribute_name}.'					)
								else:
											__UpperCAmelCase :   int    					=  arg

					if len(__UpperCAmelCase					) != len(self.attributes					):
								raise ValueError(
								    f'This processor requires {len(self.attributes					)} arguments: {", ".join(self.attributes					)}. Got '
								    f'{len(__UpperCAmelCase					)} arguments instead.'					)

					# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
					for attribute_name, arg in kwargs.items():
								__UpperCAmelCase :   int    					=  getattr(self			,							f'{attribute_name}_class'					)
								# Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
								__UpperCAmelCase :   str    					=  AUTO_TO_BASE_CLASS_MAPPING.get(__UpperCAmelCase			,							__UpperCAmelCase					)
								if isinstance(__UpperCAmelCase			,							__UpperCAmelCase					):
											__UpperCAmelCase :   str    					=  tuple(getattr(__UpperCAmelCase			,							__UpperCAmelCase					) for n in class_name if n is not None					)
								else:
											__UpperCAmelCase :   Optional[Any]    					=  getattr(__UpperCAmelCase			,							__UpperCAmelCase					)

								if not isinstance(__UpperCAmelCase			,							__UpperCAmelCase					):
											raise ValueError(
											    f'Received a {type(__UpperCAmelCase					).__name__} for argument {attribute_name}, but a {class_name} was expected.'					)

								setattr(self			,							__UpperCAmelCase			,							__UpperCAmelCase					)





		def __repr__(  self					)							->  List[str]:

					'''simple docstring'''
					__UpperCAmelCase :   Any    					=  [f'- {name}: {repr(getattr(self			,							__UpperCAmelCase					)					)}' for name in self.attributes]
					__UpperCAmelCase :   List[Any]    					=  """\n""".join(__UpperCAmelCase					)
					return f'{self.__class__.__name__}:\n{attributes_repr}'





		def 				__A							(  self			,							__UpperCAmelCase			,							__UpperCAmelCase = False			,							**__UpperCAmelCase					)							->  Any:

					'''simple docstring'''
					os.makedirs(__UpperCAmelCase			,							exist_ok=__UpperCAmelCase					)

					if push_to_hub:
								__UpperCAmelCase :   Optional[int]    					=  kwargs.pop("""commit_message"""			,							__UpperCAmelCase					)
								__UpperCAmelCase :   List[Any]    					=  kwargs.pop("""repo_id"""			,							save_directory.split(os.path.sep					)[-1]					)
								__UpperCAmelCase :   Any    					=  self._create_repo(__UpperCAmelCase			,							**__UpperCAmelCase					)
								__UpperCAmelCase :   List[str]    					=  self._get_files_timestamps(__UpperCAmelCase					)
					# If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
					# loaded from the Hub.
					if self._auto_class is not None:
								__UpperCAmelCase :   Union[str, Any]    					=  [getattr(self			,							__UpperCAmelCase					) for attribute_name in self.attributes]
								__UpperCAmelCase :   Dict    					=  [(a.init_kwargs if isinstance(__UpperCAmelCase			,							__UpperCAmelCase					) else a) for a in attrs]
								custom_object_save(self			,							__UpperCAmelCase			,							config=__UpperCAmelCase					)

					for attribute_name in self.attributes:
								__UpperCAmelCase :   Optional[int]    					=  getattr(self			,							__UpperCAmelCase					)
								# Include the processor class in the attribute config so this processor can then be reloaded with the
								# `AutoProcessor` API.
								if hasattr(__UpperCAmelCase			,							"""_set_processor_class"""					):
											attribute._set_processor_class(self.__class__.__name__					)
								attribute.save_pretrained(__UpperCAmelCase					)

					if self._auto_class is not None:
								# We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
								for attribute_name in self.attributes:
											__UpperCAmelCase :   str    					=  getattr(self			,							__UpperCAmelCase					)
											if isinstance(__UpperCAmelCase			,							__UpperCAmelCase					):
														del attribute.init_kwargs["auto_map"]

					if push_to_hub:
								self._upload_modified_files(
								    __UpperCAmelCase			,							__UpperCAmelCase			,							__UpperCAmelCase			,							commit_message=__UpperCAmelCase			,							token=kwargs.get("""use_auth_token"""					)			,							)





		@classmethod
		def 				__A							(  cls			,							__UpperCAmelCase			,							__UpperCAmelCase = None			,							__UpperCAmelCase = False			,							__UpperCAmelCase = False			,							__UpperCAmelCase = None			,							__UpperCAmelCase = "main"			,							**__UpperCAmelCase			,							)							->  Optional[int]:

					'''simple docstring'''
					__UpperCAmelCase :   List[str]    					=  cache_dir
					__UpperCAmelCase :   Tuple    					=  force_download
					__UpperCAmelCase :   Optional[int]    					=  local_files_only
					__UpperCAmelCase :   str    					=  revision

					__UpperCAmelCase :   int    					=  kwargs.pop("""use_auth_token"""			,							__UpperCAmelCase					)
					if use_auth_token is not None:
								warnings.warn(
								    """The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."""			,							__UpperCAmelCase					)
								if token is not None:
											raise ValueError(
											    """`token` and `use_auth_token` are both specified. Please set only the argument `token`."""					)
								__UpperCAmelCase :   Any    					=  use_auth_token

					if token is not None:
								# change to `token` in a follow-up PR
								__UpperCAmelCase :   Dict    					=  token

					__UpperCAmelCase :   Optional[int]    					=  cls._get_arguments_from_pretrained(__UpperCAmelCase			,							**__UpperCAmelCase					)
					return cls(*__UpperCAmelCase					)





		@classmethod
		def 				__A							(  cls			,							__UpperCAmelCase="AutoProcessor"					)							->  Dict:

					'''simple docstring'''
					if not isinstance(__UpperCAmelCase			,							__UpperCAmelCase					):
								__UpperCAmelCase :   List[Any]    					=  auto_class.__name__

					import transformers.models.auto as auto_module

					if not hasattr(__UpperCAmelCase			,							__UpperCAmelCase					):
								raise ValueError(f'{auto_class} is not a valid auto class.'					)

					__UpperCAmelCase :   Optional[Any]    					=  auto_class





		@classmethod
		def 				__A							(  cls			,							__UpperCAmelCase			,							**__UpperCAmelCase					)							->  List[str]:

					'''simple docstring'''
					__UpperCAmelCase :   Union[str, Any]    					=  []
					for attribute_name in cls.attributes:
								__UpperCAmelCase :   List[Any]    					=  getattr(cls			,							f'{attribute_name}_class'					)
								if isinstance(__UpperCAmelCase			,							__UpperCAmelCase					):
											__UpperCAmelCase :   str    					=  tuple(getattr(__UpperCAmelCase			,							__UpperCAmelCase					) if n is not None else None for n in class_name					)
											__UpperCAmelCase :   int    					=  kwargs.get("""use_fast"""			,							__UpperCAmelCase					)
											if use_fast and classes[1] is not None:
														__UpperCAmelCase :   int    					=  classes[1]
											else:
														__UpperCAmelCase :   Optional[Any]    					=  classes[0]
								else:
											__UpperCAmelCase :   str    					=  getattr(__UpperCAmelCase			,							__UpperCAmelCase					)

								args.append(attribute_class.from_pretrained(__UpperCAmelCase			,							**__UpperCAmelCase					)					)
					return args





		@property
		def 				__A							(  self					)							->  Tuple:

					'''simple docstring'''
					__UpperCAmelCase :   Optional[Any]    					=  getattr(self			,							self.attributes[0]					)
					return getattr(__UpperCAmelCase			,							"""model_input_names"""			,							__UpperCAmelCase					)


_UpperCamelCase									=      copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
					_UpperCamelCase									=      ProcessorMixin.push_to_hub.__doc__.format(
					    object='''processor''', object_class='''AutoProcessor''', object_files='''processor files'''
					)




