
'''simple docstring'''





import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
     import importlib.resources as pkg_resources
except ImportError:
     # Try backported to PY<37 `importlib_resources`.
     import importlib_resources as pkg_resources

from . import resources


A							     = "https://github.com/huggingface/datasets/tree/main/src/datasets/utils"
A							     = F"""{BASE_REF_URL}/{__file__}"""
A							     = logging.getLogger(__name__)
def 	snake_case_							(				_a  :				str      ):
      snake_case_    							=				pkg_resources.read_text(_a			,       _a      )
      return yaml.safe_load(_a      ), F'''{BASE_REF_URL}/resources/{resource}'''


A     ,  A							     = load_yaml_resource("readme_structure.yaml")

A							     = [
    "[Needs More Information]",
    "[More Information Needed]",
    "(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)",
]

# Dictionary representation of section/readme, error_list, warning_list
A							     = Tuple[dict, List[str], List[str]]



class 			lowerCAmelCase__ :



  '''simple docstring'''






  def __init__(self	:			str						,			lowerCAmelCase_	:			str						,			lowerCAmelCase_	:			str						,			lowerCAmelCase_	:			List[str] = None						,			lowerCAmelCase_	:			bool = False     ) ->							List[Any]:
        snake_case_    							=				name
        snake_case_    							=				level
        snake_case_    							=				lines
        snake_case_    							=				""
        snake_case_    							=				True
        snake_case_    							=				{}
        snake_case_    							=				[]
        snake_case_    							=				[]
        if self.lines is not None:
              self.parse(suppress_parsing_errors=lowerCAmelCase_     )

  def 		_lowerCamelCase						(self	:			int						,			lowerCAmelCase_	:			bool = False     ) ->							Union[str, Any]:
        snake_case_    							=				""
        snake_case_    							=				[]
        snake_case_    							=				False
        for line in self.lines:
              if line.strip(" \n"     ) == "":
                    continue
              elif line.strip(" \n"     )[:3] == "```":
                    snake_case_    							=				not code_start
              elif line.split()[0] == self.level + "#" and not code_start:
                    if current_sub_level != "":
                          snake_case_    							=				Section(lowerCAmelCase_						,			self.level + "#"						,			lowerCAmelCase_     )
                          snake_case_    							=				[]
                    else:
                          if current_lines != []:
                                self.text += "".join(lowerCAmelCase_     ).strip()
                                if self.text != "" and self.text not in FILLER_TEXT:
                                      snake_case_    							=				False
                                snake_case_    							=				[]

                    snake_case_    							=				" ".join(line.split()[1:]     ).strip(" \n"     )
              else:
                    current_lines.append(lowerCAmelCase_     )
        else:
              if current_sub_level != "":
                    if current_sub_level in self.content:
                          self.parsing_error_list.append(
                              f'''Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.'''     )
                    snake_case_    							=				Section(lowerCAmelCase_						,			self.level + "#"						,			lowerCAmelCase_     )
              else:
                    if current_lines != []:
                          self.text += "".join(lowerCAmelCase_     ).strip()
                          if self.text != "" and self.text not in FILLER_TEXT:
                                snake_case_    							=				False

        if self.level == "" and not suppress_parsing_errors:
              if self.parsing_error_list != [] or self.parsing_warning_list != []:
                    snake_case_    							=				snake_case_    							=				"\n".join("-\t" + x for x in self.parsing_error_list + self.parsing_warning_list     )
                    snake_case_    							=				f'''The following issues were found while parsing the README at `{self.name}`:\n''' + errors
                    raise ValueError(lowerCAmelCase_     )

  def 		_lowerCamelCase						(self	:			Dict						,			lowerCAmelCase_	:			dict     ) ->							ReadmeValidatorOutput:
        snake_case_    							=				[]
        snake_case_    							=				[]
        if structure["allow_empty"] is False:
              # If content is expected
              if self.is_empty_text and self.content == {}:
                    # If no content is found, mention it in the error_list
                    error_list.append(f'''Expected some content in section `{self.name}` but it is empty.'''     )

        if structure["allow_empty_text"] is False:
              # If some text is expected
              if self.is_empty_text:
                    # If no text is found, mention it in the error_list
                    error_list.append(
                        f'''Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).'''     )
        # Subsections Validation
        if structure["subsections"] is not None:
              # If subsections are expected
              if self.content == {}:
                    # If no subsections are present
                    snake_case_    							=				[subsection["name"] for subsection in structure["subsections"]]
                    # Mention the expected values in the error_list
                    error_list.append(
                        f'''Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values]     )}. Found \'None\'.'''     )
              else:
                    # If some subsections are present
                    snake_case_    							=				[subsection["name"] for subsection in structure["subsections"]]
                    snake_case_    							=				False
                    for idx, name in enumerate(lowerCAmelCase_     ):
                          if name not in self.content:
                                # If the expected subsection is not present
                                error_list.append(f'''Section `{self.name}` is missing subsection: `{name}`.'''     )
                                snake_case_    							=				True
                          else:
                                # If the subsection is present, validate subsection, return the result
                                # and concat the errors from subsection to section error_list

                                # Skip sublevel validation if current level is `###`
                                if self.level == "###":
                                      continue
                                else:
                                      snake_case_							,						snake_case_							,						snake_case_    							=				self.content[name].validate(
                                          structure["subsections"][idx]     )
                                error_list += subsec_error_list
                                warning_list += subsec_warning_list

                    if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
                          for name in self.content:
                                if name not in structure_names:
                                      # If an extra subsection is present
                                      warning_list.append(
                                          f'''`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.'''     )
        if error_list:
              # If there are errors, do not return the dictionary as it is invalid
              return {}, error_list, warning_list
        else:
              return self.to_dict(), error_list, warning_list


  def 		_lowerCamelCase						(self	:			Union[str, Any]     ) ->							dict:
        return {
            "name": self.name,
            "text": self.text,
            "is_empty_text": self.is_empty_text,
            "subsections": [value.to_dict() for value in self.content.values()],
        }



class 			lowerCAmelCase__ (							UpperCamelCase							):  # Level 0



  '''simple docstring'''






  def __init__(self	:			Optional[int]						,			lowerCAmelCase_	:			str						,			lowerCAmelCase_	:			List[str]						,			lowerCAmelCase_	:			dict = None						,			lowerCAmelCase_	:			bool = False     ) ->							List[str]:
        super().__init__(name=lowerCAmelCase_						,			level=""     )  # Not using lines here as we need to use a child class parse
        snake_case_    							=				structure
        snake_case_    							=				-2
        snake_case_    							=				0
        snake_case_    							=				lines
        if self.lines is not None:
              self.parse(suppress_parsing_errors=lowerCAmelCase_     )

  def 		_lowerCamelCase						(self	:			Tuple     ) ->							Any:
        if self.structure is None:
              snake_case_							,						snake_case_							,						snake_case_    							=				self._validate(lowerCAmelCase_     )
        else:
              snake_case_							,						snake_case_							,						snake_case_    							=				self._validate(self.structure     )
        if error_list != [] or warning_list != []:
              snake_case_    							=				"\n".join(["-\t" + x for x in error_list + warning_list]     )
              snake_case_    							=				f'''The following issues were found for the README at `{self.name}`:\n''' + errors
              raise ValueError(lowerCAmelCase_     )

  @classmethod
  def 		_lowerCamelCase						(cls	:			int						,			lowerCAmelCase_	:			Path						,			lowerCAmelCase_	:			dict = None						,			lowerCAmelCase_	:			bool = False     ) ->							int:
        with open(lowerCAmelCase_						,			encoding="utf-8"     ) as f:
              snake_case_    							=				f.readlines()
        return cls(lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_						,			suppress_parsing_errors=lowerCAmelCase_     )

  @classmethod
  def 		_lowerCamelCase						(cls	:			Optional[int]						,			lowerCAmelCase_	:			str						,			lowerCAmelCase_	:			dict = None						,			lowerCAmelCase_	:			str = "root"						,			lowerCAmelCase_	:			bool = False     ) ->							Optional[int]:
        snake_case_    							=				string.split("\n"     )
        return cls(lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_						,			suppress_parsing_errors=lowerCAmelCase_     )

  def 		_lowerCamelCase						(self	:			Optional[Any]						,			lowerCAmelCase_	:			bool = False     ) ->							Dict:
        # Skip Tags
        snake_case_    							=				0

        for line in self.lines:
              self.yaml_tags_line_count += 1
              if line.strip(" \n"     ) == "---":
                    self.tag_count += 1
                    if self.tag_count == 2:
                          break
              line_count += 1
        if self.tag_count == 2:
              snake_case_    							=				self.lines[line_count + 1 :]  # Get the last + 1 th item.
        else:
              snake_case_    							=				self.lines[self.tag_count :]
        super().parse(suppress_parsing_errors=lowerCAmelCase_     )

  def __str__(self	:			Any     ) ->							Any:
        return str(self.to_dict()     )


  def 		_lowerCamelCase						(self	:			Union[str, Any]						,			lowerCAmelCase_	:			Tuple     ) ->							Optional[Any]:
        snake_case_    							=				[]
        snake_case_    							=				[]
        if self.yaml_tags_line_count == 0:
              warning_list.append("Empty YAML markers are present in the README."     )
        elif self.tag_count == 0:
              warning_list.append("No YAML markers are present in the README."     )
        elif self.tag_count == 1:
              warning_list.append("Only the start of YAML tags present in the README."     )
        # Check how many first level sections are present.
        snake_case_    							=				len(self.content.keys()     )
        if num_first_level_keys > 1:
              # If more than one, add to the error list, continue
              error_list.append(
                  f'''The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys()     )]     )}. Only one heading is expected. Skipping further validation for this README.'''     )
        elif num_first_level_keys < 1:
              # If less than one, append error.
              error_list.append(
                  "The README has no first-level headings. One heading is expected. Skipping further validation for this README."     )

        else:
              # If one exactly
              snake_case_    							=				list(self.content.keys()     )[0]  # Get the key
              if start_key.startswith("Dataset Card for"     ):  # Check correct start
                    # If the starting is correct, validate all the sections
                    snake_case_							,						snake_case_							,						snake_case_    							=				self.content[start_key].validate(
                        readme_structure["subsections"][0]     )
                    error_list += sec_error_list
                    warning_list += sec_warning_list
              else:
                    # If not found, append error
                    error_list.append(
                        "No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."     )
        if error_list:
              # If there are errors, do not return the dictionary as it is invalid
              return {}, error_list, warning_list
        else:
              return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
     from argparse import ArgumentParser

     A							     = ArgumentParser(usage="Validate the content (excluding YAML tags) of a README.md file.")
     ap.add_argument("readme_filepath")
     A							     = ap.parse_args()
     A							     = Path(args.readme_filepath)
     A							     = ReadMe.from_readme(readme_filepath)






import argparse
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_dummies.py
UpperCamelCase__			=    "src/transformers"

# Matches is_xxx_available()
UpperCamelCase__			=    re.compile(R"is\_([a-z_]*)_available()")
# Matches from xxx import bla
UpperCamelCase__			=    re.compile(R"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")
UpperCamelCase__			=    re.compile(R"^\s+if\s+not\s+\(?is\_[a-z_]*\_available\(\)")


UpperCamelCase__			=    "\n{0} = None\n"

UpperCamelCase__			=    "\nclass {0}(metaclass=DummyObject):\n    _backends = {1}\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, {1})\n"


UpperCamelCase__			=    "\ndef {0}(*args, **kwargs):\n    requires_backends({0}, {1})\n"






def 			UpperCAmelCase__     (     lowercase__ )			->   List[Any]:
						if _re_test_backend.search(lowercase__ ) is None:
												return None
						__lowercase											=					[b[0] for b in _re_backend.findall(lowercase__ )]
						backends.sort()
						return "_and_".join(lowercase__ )






def 			UpperCAmelCase__     (     )			->   Dict:
						with open(os.path.join(lowercase__    ,							"""__init__.py""" )    ,							"""r"""    ,							encoding="""utf-8"""    ,							newline="""\n""" ) as f:
												__lowercase											=					f.readlines()

						# Get to the point we do the actual imports for type checking
						__lowercase											=					0
						while not lines[line_index].startswith("""if TYPE_CHECKING""" ):
												line_index += 1

						__lowercase											=					{}
						# Go through the end of the file
						while line_index < len(lowercase__ ):
												# If the line is an if is_backend_available, we grab all objects associated.
												__lowercase											=					find_backend(lines[line_index] )
												if backend is not None:
																		while not lines[line_index].startswith("""    else:""" ):
																								line_index += 1
																		line_index += 1

																		__lowercase											=					[]
																		# Until we unindent, add backend objects to the list
																		while len(lines[line_index] ) <= 1 or lines[line_index].startswith(""" """ * 8 ):
																								__lowercase											=					lines[line_index]
																								__lowercase											=					_re_single_line_import.search(lowercase__ )
																								if single_line_import_search is not None:
																														objects.extend(single_line_import_search.groups()[0].split(""", """ ) )
																								elif line.startswith(""" """ * 12 ):
																														objects.append(line[12:-2] )
																								line_index += 1

																		__lowercase											=					objects
												else:
																		line_index += 1

						return backend_specific_objects






def 			UpperCAmelCase__     (     lowercase__    ,							lowercase__ )			->   int:
						if name.isupper():
												return DUMMY_CONSTANT.format(lowercase__ )
						elif name.islower():
												return DUMMY_FUNCTION.format(lowercase__    ,							lowercase__ )
						else:
												return DUMMY_CLASS.format(lowercase__    ,							lowercase__ )






def 			UpperCAmelCase__     (     lowercase__=None )			->   List[str]:
						if backend_specific_objects is None:
												__lowercase											=					read_init()
						# For special correspondence backend to module name as used in the function requires_modulename
						__lowercase											=					{}

						for backend, objects in backend_specific_objects.items():
												__lowercase											=					"""[""" + """, """.join(F"\"{b}\"" for b in backend.split("""_and_""" ) ) + """]"""
												__lowercase											=					"""# This file is autogenerated by the command `make fix-copies`, do not edit.\n"""
												dummy_file += "from ..utils import DummyObject, requires_backends\n\n"
												dummy_file += "\n".join([create_dummy_object(lowercase__    ,							lowercase__ ) for o in objects] )
												__lowercase											=					dummy_file

						return dummy_files






def 			UpperCAmelCase__     (     lowercase__=False )			->   Optional[Any]:
						__lowercase											=					create_dummy_files()
						# For special correspondence backend to shortcut as used in utils/dummy_xxx_objects.py
						__lowercase											=					{"""torch""": """pt"""}

						# Locate actual dummy modules and read their content.
						__lowercase											=					os.path.join(lowercase__    ,							"""utils""" )
						__lowercase											=					{
						    backend: os.path.join(lowercase__    ,							F"dummy_{short_names.get(lowercase__    ,							lowercase__ )}_objects.py" )
						    for backend in dummy_files.keys()
						}

						__lowercase											=					{}
						for backend, file_path in dummy_file_paths.items():
												if os.path.isfile(lowercase__ ):
																		with open(lowercase__    ,							"""r"""    ,							encoding="""utf-8"""    ,							newline="""\n""" ) as f:
																								__lowercase											=					f.read()
												else:
																		__lowercase											=					""""""

						for backend in dummy_files.keys():
												if dummy_files[backend] != actual_dummies[backend]:
																		if overwrite:
																								print(
																								    F"Updating transformers.utils.dummy_{short_names.get(lowercase__    ,							lowercase__ )}_objects.py as the main "
																								    """__init__ has new objects.""" )
																								with open(dummy_file_paths[backend]    ,							"""w"""    ,							encoding="""utf-8"""    ,							newline="""\n""" ) as f:
																														f.write(dummy_files[backend] )
																		else:
																								raise ValueError(
																								    """The main __init__ has objects that are not present in """
																								    F"transformers.utils.dummy_{short_names.get(lowercase__    ,							lowercase__ )}_objects.py. Run `make fix-copies` "
																								    """to fix this.""" )


if __name__ == "__main__":
			UpperCamelCase__			=    argparse.ArgumentParser()
			parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
			UpperCamelCase__			=    parser.parse_args()

			check_dummies(args.fix_and_overwrite)





