




'''simple docstring'''



import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


UpperCAmelCase_       :  List[Any]      =		get_logger(__name__)

# Datasets
UpperCAmelCase_       :  Dict      =		'''https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets'''
UpperCAmelCase_       :  Dict      =		'''https://cdn-datasets.huggingface.co/datasets/datasets'''
UpperCAmelCase_       :  List[Any]      =		'''https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}'''

# Metrics
UpperCAmelCase_       :  int      =		'''https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics'''
UpperCAmelCase_       :  str      =		'''https://cdn-datasets.huggingface.co/datasets/metric'''
UpperCAmelCase_       :  Optional[Any]      =		'''https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}'''

# Hub
UpperCAmelCase_       :  Dict      =		os.environ.get('''HF_ENDPOINT''', '''https://huggingface.co''')
UpperCAmelCase_       :  Tuple      =		HF_ENDPOINT + '''/datasets/{repo_id}/resolve/{revision}/{path}'''
UpperCAmelCase_       :  Any      =		'''main'''

UpperCAmelCase_       :  str      =		version.parse(platform.python_version())

if PY_VERSION < version.parse('''3.8'''):
     import importlib_metadata
else:
     import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
UpperCAmelCase_       :  Optional[Any]      =		{'''1''', '''ON''', '''YES''', '''TRUE'''}
UpperCAmelCase_       :  Dict      =		ENV_VARS_TRUE_VALUES.union({'''AUTO'''})


# Imports
UpperCAmelCase_       :  Any      =		version.parse(importlib_metadata.version('''dill'''))
UpperCAmelCase_       :  List[str]      =		version.parse(importlib_metadata.version('''pandas'''))
UpperCAmelCase_       :  Optional[Any]      =		version.parse(importlib_metadata.version('''pyarrow'''))

UpperCAmelCase_       :  Tuple      =		os.environ.get('''USE_TF''', '''AUTO''').upper()
UpperCAmelCase_       :  Dict      =		os.environ.get('''USE_TORCH''', '''AUTO''').upper()
UpperCAmelCase_       :  Tuple      =		os.environ.get('''USE_JAX''', '''AUTO''').upper()

UpperCAmelCase_       :  int      =		'''N/A'''
UpperCAmelCase_       :  List[Any]      =		False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
     UpperCAmelCase_       :  List[Any]      =		importlib.util.find_spec('''torch''') is not None
     if TORCH_AVAILABLE:
          try:
               UpperCAmelCase_       :  Tuple      =		version.parse(importlib_metadata.version('''torch'''))
               logger.info(F"""PyTorch version {TORCH_VERSION} available.""")
          except importlib_metadata.PackageNotFoundError:
               pass
else:
     logger.info('''Disabling PyTorch because USE_TF is set''')

UpperCAmelCase_       :  str      =		'''N/A'''
UpperCAmelCase_       :  Optional[Any]      =		False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
     UpperCAmelCase_       :  List[Any]      =		importlib.util.find_spec('''tensorflow''') is not None
     if TF_AVAILABLE:
          # For the metadata, we have to look for both tensorflow and tensorflow-cpu
          for package in [
              "tensorflow",
              "tensorflow-cpu",
              "tensorflow-gpu",
              "tf-nightly",
              "tf-nightly-cpu",
              "tf-nightly-gpu",
              "intel-tensorflow",
              "tensorflow-rocm",
              "tensorflow-macos",
          ]:
               try:
                    UpperCAmelCase_       :  int      =		version.parse(importlib_metadata.version(package))
               except importlib_metadata.PackageNotFoundError:
                    continue
               else:
                    break
          else:
               UpperCAmelCase_       :  Union[str, Any]      =		False
     if TF_AVAILABLE:
          if TF_VERSION.major < 2:
               logger.info(F"""TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.""")
               UpperCAmelCase_       :  Dict      =		False
          else:
               logger.info(F"""TensorFlow version {TF_VERSION} available.""")
else:
     logger.info('''Disabling Tensorflow because USE_TORCH is set''')


UpperCAmelCase_       :  str      =		'''N/A'''
UpperCAmelCase_       :  Optional[int]      =		False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
     UpperCAmelCase_       :  str      =		importlib.util.find_spec('''jax''') is not None and importlib.util.find_spec('''jaxlib''') is not None
     if JAX_AVAILABLE:
          try:
               UpperCAmelCase_       :  Union[str, Any]      =		version.parse(importlib_metadata.version('''jax'''))
               logger.info(F"""JAX version {JAX_VERSION} available.""")
          except importlib_metadata.PackageNotFoundError:
               pass
else:
     logger.info('''Disabling JAX because USE_JAX is set to False''')


UpperCAmelCase_       :  Tuple      =		os.environ.get('''USE_BEAM''', '''AUTO''').upper()
UpperCAmelCase_       :  Dict      =		'''N/A'''
UpperCAmelCase_       :  Any      =		False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
     try:
          UpperCAmelCase_       :  Optional[int]      =		version.parse(importlib_metadata.version('''apache_beam'''))
          UpperCAmelCase_       :  Any      =		True
          logger.info(F"""Apache Beam version {BEAM_VERSION} available.""")
     except importlib_metadata.PackageNotFoundError:
          pass
else:
     logger.info('''Disabling Apache Beam because USE_BEAM is set to False''')


# Optional tools for data loading
UpperCAmelCase_       :  str      =		importlib.util.find_spec('''sqlalchemy''') is not None

# Optional tools for feature decoding
UpperCAmelCase_       :  Dict      =		importlib.util.find_spec('''PIL''') is not None
UpperCAmelCase_       :  Optional[Any]      =		importlib.util.find_spec('''soundfile''') is not None and version.parse(
    importlib.import_module('''soundfile''').__libsndfile_version__
) >= version.parse('''1.0.31''')
UpperCAmelCase_       :  Optional[int]      =		importlib.util.find_spec('''soundfile''') is not None and version.parse(
    importlib.import_module('''soundfile''').__libsndfile_version__
) >= version.parse('''1.1.0''')

# Optional compression tools
UpperCAmelCase_       :  int      =		importlib.util.find_spec('''rarfile''') is not None
UpperCAmelCase_       :  int      =		importlib.util.find_spec('''zstandard''') is not None
UpperCAmelCase_       :  List[str]      =		importlib.util.find_spec('''lz4''') is not None
UpperCAmelCase_       :  int      =		importlib.util.find_spec('''py7zr''') is not None


# Cache location
UpperCAmelCase_       :  Dict      =		'''~/.cache'''
UpperCAmelCase_       :  Union[str, Any]      =		os.getenv('''XDG_CACHE_HOME''', DEFAULT_XDG_CACHE_HOME)
UpperCAmelCase_       :  Optional[Any]      =		os.path.join(XDG_CACHE_HOME, '''huggingface''')
UpperCAmelCase_       :  Optional[Any]      =		os.path.expanduser(os.getenv('''HF_HOME''', DEFAULT_HF_CACHE_HOME))

UpperCAmelCase_       :  Dict      =		os.path.join(HF_CACHE_HOME, '''datasets''')
UpperCAmelCase_       :  Tuple      =		Path(os.getenv('''HF_DATASETS_CACHE''', DEFAULT_HF_DATASETS_CACHE))

UpperCAmelCase_       :  List[Any]      =		os.path.join(HF_CACHE_HOME, '''metrics''')
UpperCAmelCase_       :  Dict      =		Path(os.getenv('''HF_METRICS_CACHE''', DEFAULT_HF_METRICS_CACHE))

UpperCAmelCase_       :  str      =		os.path.join(HF_CACHE_HOME, '''modules''')
UpperCAmelCase_       :  List[str]      =		Path(os.getenv('''HF_MODULES_CACHE''', DEFAULT_HF_MODULES_CACHE))

UpperCAmelCase_       :  Dict      =		'''downloads'''
UpperCAmelCase_       :  Optional[int]      =		os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
UpperCAmelCase_       :  Tuple      =		Path(os.getenv('''HF_DATASETS_DOWNLOADED_DATASETS_PATH''', DEFAULT_DOWNLOADED_DATASETS_PATH))

UpperCAmelCase_       :  Optional[Any]      =		'''extracted'''
UpperCAmelCase_       :  List[str]      =		os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
UpperCAmelCase_       :  Optional[int]      =		Path(os.getenv('''HF_DATASETS_EXTRACTED_DATASETS_PATH''', DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
UpperCAmelCase_       :  Optional[Any]      =		(
    os.environ.get('''HF_UPDATE_DOWNLOAD_COUNTS''', '''AUTO''').upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
UpperCAmelCase_       :  Union[str, Any]      =		1_0_0_0

# Size of the preloaded record batch in `Dataset.__iter__`
UpperCAmelCase_       :  List[str]      =		1_0

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
UpperCAmelCase_       :  List[Any]      =		4 << 3_0

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
UpperCAmelCase_       :  Any      =		'''500MB'''

# Parquet configuration
UpperCAmelCase_       :  Optional[int]      =		1_0_0
UpperCAmelCase_       :  List[Any]      =		1_0_0
UpperCAmelCase_       :  Tuple      =		1_0_0

# Offline mode
UpperCAmelCase_       :  List[Any]      =		os.environ.get('''HF_DATASETS_OFFLINE''', '''AUTO''').upper() in ENV_VARS_TRUE_VALUES

# In-memory
UpperCAmelCase_       :  Any      =		0  # Disabled
UpperCAmelCase_       :  str      =		float(os.environ.get('''HF_DATASETS_IN_MEMORY_MAX_SIZE''', DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
UpperCAmelCase_       :  Optional[int]      =		'''dataset.arrow'''
UpperCAmelCase_       :  Dict      =		'''indices.arrow'''
UpperCAmelCase_       :  Dict      =		'''state.json'''
UpperCAmelCase_       :  Optional[int]      =		'''dataset_info.json'''
UpperCAmelCase_       :  Dict      =		'''dataset_infos.json'''
UpperCAmelCase_       :  List[str]      =		'''LICENSE'''
UpperCAmelCase_       :  str      =		'''metric_info.json'''
UpperCAmelCase_       :  str      =		'''dataset_dict.json'''

UpperCAmelCase_       :  Tuple      =		'''datasets_modules'''

UpperCAmelCase_       :  Union[str, Any]      =		2_5_5

# Streaming
UpperCAmelCase_       :  Union[str, Any]      =		2_0
UpperCAmelCase_       :  List[Any]      =		5

# Datasets without script
UpperCAmelCase_       :  Dict      =		2_0_0
UpperCAmelCase_       :  List[Any]      =		1_0
UpperCAmelCase_       :  Optional[Any]      =		2_0_0

# Progress bars
UpperCAmelCase_       :  str      =		0.05  # 20 progress updates per sec












'''simple docstring'''



# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import List







def 			_UpperCamelCase (_lowerCamelCase							:     List[str]   ,	_lowerCamelCase							:     str	)->   str:




     '''simple docstring'''





     if name != "training_function" and name != "main":
          raise ValueError(f'''Incorrect function name passed: {name}, choose either \'main\' or \'training_function\''''	)
     __snake_case					,    __snake_case    					=  [], False
     for line in lines:
          if not found_start and f'''def {name}''' in line:
               __snake_case    					=  True
               good_lines.append(_lowerCamelCase	)
               continue
          if found_start:
               if name == "training_function" and "def main" in line:
                    return good_lines
               if name == "main" and "if __name__" in line:
                    return good_lines
               good_lines.append(_lowerCamelCase	)







def 			_UpperCamelCase (_lowerCamelCase							:     List[str]	)->   int:




     '''simple docstring'''





     return [line for line in lines if not line.lstrip().startswith('''#'''	) and line != "\n"]







def 			_UpperCamelCase (_lowerCamelCase							:     str   ,	_lowerCamelCase							:     str   ,	_lowerCamelCase							:     bool   ,	_lowerCamelCase							:     str = None	)->   Optional[Any]:




     '''simple docstring'''





     with open(_lowerCamelCase   ,	'''r'''	) as f:
          __snake_case    					=  f.readlines()
     with open(os.path.abspath(os.path.join('''examples'''   ,	'''nlp_example.py'''	)	)   ,	'''r'''	) as f:
          __snake_case    					=  f.readlines()
     with open(_lowerCamelCase   ,	'''r'''	) as f:
          __snake_case    					=  f.readlines()
     if secondary_filename is not None:
          with open(_lowerCamelCase   ,	'''r'''	) as f:
               __snake_case    					=  f.readlines()

    # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content
     if parser_only:
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''main'''	)	)
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''main'''	)	)
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''main'''	)	)
          if secondary_filename is not None:
               __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''main'''	)	)
     else:
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''training_function'''	)	)
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''training_function'''	)	)
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''training_function'''	)	)
          if secondary_filename is not None:
               __snake_case    					=  clean_lines(
                   get_function_contents_by_name(_lowerCamelCase   ,	'''training_function'''	)	)

     __snake_case    					=  '''train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n'''

     # Specific code in our script that differs from the full version, aka what is new
     __snake_case    					=  []
     __snake_case    					=  []  # We keep track of the idxs just in case it's a repeated statement
     __snake_case    					=  iter(_lowerCamelCase	)
     for i in range(len(_lowerCamelCase	) - 1	):
          if i not in passed_idxs:
               __snake_case    					=  next(_lowerCamelCase	)
               if (line not in full_file_func) and (line.lstrip() != _dl_line):
                    if "TESTING_MOCKED_DATALOADERS" not in line:
                         new_feature_code.append(_lowerCamelCase	)
                         passed_idxs.append(_lowerCamelCase	)
                    else:
                         # Skip over the `config['num_epochs'] = 2` statement
                         __snake_case    					=  next(_lowerCamelCase	)

    # Extract out just the new parts from the full_file_training_func
     __snake_case    					=  []
     __snake_case    					=  []  # We keep track of the idxs just in case it's a repeated statement
     for i, line in enumerate(_lowerCamelCase	):
          if i not in passed_idxs:
               if (line not in full_file_func) and (line.lstrip() != _dl_line):
                    if "TESTING_MOCKED_DATALOADERS" not in line:
                         new_full_example_parts.append(_lowerCamelCase	)
                         passed_idxs.append(_lowerCamelCase	)

    # Finally, get the overall diff
     __snake_case    					=  [line for line in new_feature_code if line not in new_full_example_parts]
     if secondary_filename is not None:
          __snake_case    					=  [line for line in full_file_contents if line not in secondary_file_func]
          __snake_case    					=  [line for line in diff_from_example if line not in diff_from_two]

     return diff_from_example






