



import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
   import importlib.resources as pkg_resources
except ImportError:
   # Try backported to PY<37 `importlib_resources`.
   import importlib_resources as pkg_resources

from . import resources


__lowerCamelCase								=      '''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
__lowerCamelCase								=      F'''{BASE_REF_URL}/{__file__}'''
__lowerCamelCase								=      logging.getLogger(__name__)







def  _snake_case			(							__snake_case     )  ->  List[str]:






      '''simple docstring'''







      UpperCAmelCase_	:						Optional[int]					   =		pkg_resources.read_text(_lowercase       ,	_lowercase     )
      return yaml.safe_load(_lowercase     ), F"""{BASE_REF_URL}/resources/{resource}"""


__lowerCamelCase							,  __lowerCamelCase								=      load_yaml_resource('''readme_structure.yaml''')

__lowerCamelCase								=      [
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
__lowerCamelCase								=      Tuple[dict, List[str], List[str]]


class 	snake_case_			:




       """simple docstring"""






       def __init__(  self  ,lowercase  ,lowercase  ,lowercase = None  ,lowercase = False):

             """simple docstring"""





             UpperCAmelCase_	:						List[str]					   =		name
             UpperCAmelCase_	:						Union[str, Any]					   =		level
             UpperCAmelCase_	:						Tuple					   =		lines
             UpperCAmelCase_	:						Union[str, Any]					   =		""""""
             UpperCAmelCase_	:						Union[str, Any]					   =		True
             UpperCAmelCase_	:						Any					   =		{}
             UpperCAmelCase_	:						Optional[int]					   =		[]
             UpperCAmelCase_	:						Union[str, Any]					   =		[]
             if self.lines is not None:
                   self.parse(suppress_parsing_errors=lowercase)



       def 			A_ (  self  ,lowercase = False):

             """simple docstring"""





             UpperCAmelCase_	:						Union[str, Any]					   =		""""""
             UpperCAmelCase_	:						int					   =		[]
             UpperCAmelCase_	:						Dict					   =		False
             for line in self.lines:
                   if line.strip(" \n") == "":
                         continue
                   elif line.strip(" \n")[:3] == "```":
                         UpperCAmelCase_	:						Dict					   =		not code_start
                   elif line.split()[0] == self.level + "#" and not code_start:
                         if current_sub_level != "":
                               UpperCAmelCase_	:						Any					   =		Section(lowercase  ,self.level + "#"  ,lowercase)
                               UpperCAmelCase_	:						List[Any]					   =		[]
                         else:
                               if current_lines != []:
                                     self.text += "".join(lowercase).strip()
                                     if self.text != "" and self.text not in FILLER_TEXT:
                                           UpperCAmelCase_	:						Optional[int]					   =		False
                                     UpperCAmelCase_	:						Tuple					   =		[]

                         UpperCAmelCase_	:						Union[str, Any]					   =		""" """.join(line.split()[1:]).strip(" \n")
                   else:
                         current_lines.append(lowercase)
             else:
                   if current_sub_level != "":
                         if current_sub_level in self.content:
                               self.parsing_error_list.append(
                                   F"""Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.""")
                         UpperCAmelCase_	:						Dict					   =		Section(lowercase  ,self.level + "#"  ,lowercase)
                   else:
                         if current_lines != []:
                               self.text += "".join(lowercase).strip()
                               if self.text != "" and self.text not in FILLER_TEXT:
                                     UpperCAmelCase_	:						Optional[int]					   =		False

             if self.level == "" and not suppress_parsing_errors:
                   if self.parsing_error_list != [] or self.parsing_warning_list != []:
                         UpperCAmelCase_	:						Optional[Any]					   =		"""\n""".join("-\t" + x for x in self.parsing_error_list + self.parsing_warning_list)
                         UpperCAmelCase_	:						List[Any]					   =		F"""The following issues were found while parsing the README at `{self.name}`:\n""" + errors
                         raise ValueError(lowercase)



       def 			A_ (  self  ,lowercase):

             """simple docstring"""





             UpperCAmelCase_	:						Tuple					   =		[]
             UpperCAmelCase_	:						str					   =		[]
             if structure["allow_empty"] is False:
                   # If content is expected
                   if self.is_empty_text and self.content == {}:
                         # If no content is found, mention it in the error_list
                         error_list.append(F"""Expected some content in section `{self.name}` but it is empty.""")

             if structure["allow_empty_text"] is False:
                   # If some text is expected
                   if self.is_empty_text:
                         # If no text is found, mention it in the error_list
                         error_list.append(
                             F"""Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).""")
        # Subsections Validation
             if structure["subsections"] is not None:
                   # If subsections are expected
                   if self.content == {}:
                         # If no subsections are present
                         UpperCAmelCase_	:						List[str]					   =		[subsection["""name"""] for subsection in structure["""subsections"""]]
                         # Mention the expected values in the error_list
                         error_list.append(
                             F"""Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values])}. Found 'None'.""")
                   else:
                         # If some subsections are present
                         UpperCAmelCase_	:						List[Any]					   =		[subsection["""name"""] for subsection in structure["""subsections"""]]
                         UpperCAmelCase_	:						Union[str, Any]					   =		False
                         for idx, name in enumerate(lowercase):
                               if name not in self.content:
                                     # If the expected subsection is not present
                                     error_list.append(F"""Section `{self.name}` is missing subsection: `{name}`.""")
                                     UpperCAmelCase_	:						Any					   =		True
                               else:
                                     # If the subsection is present, validate subsection, return the result
                                     # and concat the errors from subsection to section error_list

                                     # Skip sublevel validation if current level is `###`
                                     if self.level == "###":
                                           continue
                                     else:
                                           UpperCAmelCase_	:						Dict					   =		self.content[name].validate(
                                               structure["subsections"][idx])
                                     error_list += subsec_error_list
                                     warning_list += subsec_warning_list

                         if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
                               for name in self.content:
                                     if name not in structure_names:
                                           # If an extra subsection is present
                                           warning_list.append(
                                               F"""`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.""")
             if error_list:
                   # If there are errors, do not return the dictionary as it is invalid
                   return {}, error_list, warning_list
             else:
                   return self.to_dict(), error_list, warning_list



       def 			A_ (  self):

             """simple docstring"""





             return {
                 "name": self.name,
                 "text": self.text,
                 "is_empty_text": self.is_empty_text,
                 "subsections": [value.to_dict() for value in self.content.values()],
             }


class 	snake_case_			(_lowerCAmelCase	):  # Level 0




       """simple docstring"""






       def __init__(  self  ,lowercase  ,lowercase  ,lowercase = None  ,lowercase = False):

             """simple docstring"""





             super().__init__(name=lowercase  ,level="")  # Not using lines here as we need to use a child class parse
             UpperCAmelCase_	:						List[str]					   =		structure
             UpperCAmelCase_	:						List[str]					   =		-2
             UpperCAmelCase_	:						Tuple					   =		0
             UpperCAmelCase_	:						Tuple					   =		lines
             if self.lines is not None:
                   self.parse(suppress_parsing_errors=lowercase)



       def 			A_ (  self):

             """simple docstring"""





             if self.structure is None:
                   UpperCAmelCase_	:						int					   =		self._validate(lowercase)
             else:
                   UpperCAmelCase_	:						Tuple					   =		self._validate(self.structure)
             if error_list != [] or warning_list != []:
                   UpperCAmelCase_	:						str					   =		"""\n""".join(["-\t" + x for x in error_list + warning_list])
                   UpperCAmelCase_	:						Any					   =		F"""The following issues were found for the README at `{self.name}`:\n""" + errors
                   raise ValueError(lowercase)



       @classmethod
       def 			A_ (  cls  ,lowercase  ,lowercase = None  ,lowercase = False):

             """simple docstring"""





             with open(lowercase  ,encoding="utf-8") as f:
                   UpperCAmelCase_	:						List[Any]					   =		f.readlines()
             return cls(lowercase  ,lowercase  ,lowercase  ,suppress_parsing_errors=lowercase)



       @classmethod
       def 			A_ (  cls  ,lowercase  ,lowercase = None  ,lowercase = "root"  ,lowercase = False):

             """simple docstring"""





             UpperCAmelCase_	:						List[Any]					   =		string.split("\n")
             return cls(lowercase  ,lowercase  ,lowercase  ,suppress_parsing_errors=lowercase)



       def 			A_ (  self  ,lowercase = False):

             """simple docstring"""





             UpperCAmelCase_	:						Optional[Any]					   =		0

             for line in self.lines:
                   self.yaml_tags_line_count += 1
                   if line.strip(" \n") == "---":
                         self.tag_count += 1
                         if self.tag_count == 2:
                               break
                   line_count += 1
             if self.tag_count == 2:
                   UpperCAmelCase_	:						List[Any]					   =		self.lines[line_count + 1 :]  # Get the last + 1 th item.
             else:
                   UpperCAmelCase_	:						Optional[int]					   =		self.lines[self.tag_count :]
             super().parse(suppress_parsing_errors=lowercase)



       def __str__(  self):

             """simple docstring"""





             return str(self.to_dict())



       def 			A_ (  self  ,lowercase):

             """simple docstring"""





             UpperCAmelCase_	:						Dict					   =		[]
             UpperCAmelCase_	:						List[str]					   =		[]
             if self.yaml_tags_line_count == 0:
                   warning_list.append("Empty YAML markers are present in the README.")
             elif self.tag_count == 0:
                   warning_list.append("No YAML markers are present in the README.")
             elif self.tag_count == 1:
                   warning_list.append("Only the start of YAML tags present in the README.")
             # Check how many first level sections are present.
             UpperCAmelCase_	:						Optional[Any]					   =		len(self.content.keys())
             if num_first_level_keys > 1:
                   # If more than one, add to the error list, continue
                   error_list.append(
                       F"""The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys())])}. Only one heading is expected. Skipping further validation for this README.""")
             elif num_first_level_keys < 1:
                   # If less than one, append error.
                   error_list.append(
                       "The README has no first-level headings. One heading is expected. Skipping further validation for this README.")

             else:
                   # If one exactly
                   UpperCAmelCase_	:						Any					   =		list(self.content.keys())[0]  # Get the key
                   if start_key.startswith("Dataset Card for"):  # Check correct start
                         # If the starting is correct, validate all the sections
                         UpperCAmelCase_	:						List[Any]					   =		self.content[start_key].validate(
                             readme_structure["subsections"][0])
                         error_list += sec_error_list
                         warning_list += sec_warning_list
                   else:
                         # If not found, append error
                         error_list.append(
                             "No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.")
             if error_list:
                   # If there are errors, do not return the dictionary as it is invalid
                   return {}, error_list, warning_list
             else:
                   return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
   from argparse import ArgumentParser

   __lowerCamelCase								=      ArgumentParser(usage='''Validate the content (excluding YAML tags) of a README.md file.''')
   ap.add_argument('''readme_filepath''')
   __lowerCamelCase								=      ap.parse_args()
   __lowerCamelCase								=      Path(args.readme_filepath)
   __lowerCamelCase								=      ReadMe.from_readme(readme_filepath)





"""simple docstring"""
import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp
def   A_    (     ):







						'''simple docstring'''


						snake_case_					:str		=     ArgumentParser(
						    description=(
						        """PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"""
						    )      )

						# Optional arguments for the launch helper
						parser.add_argument("""--num_cores""",   type=_lowercase,   default=1,   help="""Number of TPU cores to use (1 or 8)."""      )

						# positional
						parser.add_argument(
						    """training_script""",   type=_lowercase,   help=(
						        """The full path to the single TPU training """
						        """program/script to be launched in parallel, """
						        """followed by all the arguments for the """
						        """training script"""
						    ),   )

						# rest from the training program
						parser.add_argument("""training_script_args""",   nargs=_lowercase      )

						return parser.parse_args()
def   A_    (     ):







						'''simple docstring'''


						snake_case_					:Any		=     parse_args()

						# Import training_script as a module.
						snake_case_					:str		=     Path(args.training_script      )
						sys.path.append(str(script_fpath.parent.resolve()      )      )
						snake_case_					:Dict		=     script_fpath.stem
						snake_case_					:int		=     importlib.import_module(_lowercase      )

						# Patch sys.argv
						snake_case_					:Optional[Any]		=     [args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores      )]

						xmp.spawn(mod._mp_fn,   args=(),   nprocs=args.num_cores      )


if __name__ == "__main__":
						main()
