



"""simple docstring"""
import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
						import importlib.resources as pkg_resources
except ImportError:
						# Try backported to PY<37 `importlib_resources`.
						import importlib_resources as pkg_resources

from . import resources


__a           =							"https://github.com/huggingface/datasets/tree/main/src/datasets/utils"
__a           =							F"""{BASE_REF_URL}/{__file__}"""
__a           =							logging.getLogger(__name__)
def   A_    (     _lowercase      ):







						'''simple docstring'''


						snake_case_					:Optional[int]		=     pkg_resources.read_text(_lowercase,   _lowercase      )
						return yaml.safe_load(_lowercase      ), f"""{BASE_REF_URL}/resources/{resource}"""


__a       ,	__a           =							load_yaml_resource("readme_structure.yaml")

__a           =							[
    "[Needs More Information]",
    "[More Information Needed]",
    "(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)",
]

# Dictionary representation of section/readme, error_list, warning_list
__a           =							Tuple[dict, List[str], List[str]]
class 							lowerCamelCase    :






					'''simple docstring'''





					def __init__(			self:				Optional[Any]		,	snake_case:				str		,	snake_case:				str		,	snake_case:				List[str] = None		,	snake_case:				bool = False		)  ->      Optional[Any]:
											snake_case_					:List[str]		=     name
											snake_case_					:Union[str, Any]		=     level
											snake_case_					:Tuple		=     lines
											snake_case_					:Union[str, Any]		=     """"""
											snake_case_					:Union[str, Any]		=     True
											snake_case_					:Any		=     {}
											snake_case_					:Optional[int]		=     []
											snake_case_					:Union[str, Any]		=     []
											if self.lines is not None:
																	self.parse(suppress_parsing_errors=snake_case		)






					def   lowerCAmelCase_							(			self:				Optional[int]		,	snake_case:				bool = False		)  ->      Tuple:
											snake_case_					:Union[str, Any]		=     """"""
											snake_case_					:int		=     []
											snake_case_					:Dict		=     False
											for line in self.lines:
																	if line.strip(""" \n"""		) == "":
																							continue
																	elif line.strip(""" \n"""		)[:3] == "```":
																							snake_case_					:Dict		=     not code_start
																	elif line.split()[0] == self.level + "#" and not code_start:
																							if current_sub_level != "":
																													snake_case_					:Any		=     Section(snake_case		,	self.level + """#"""		,	snake_case		)
																													snake_case_					:List[Any]		=     []
																							else:
																													if current_lines != []:
																																			self.text += "".join(snake_case		).strip()
																																			if self.text != "" and self.text not in FILLER_TEXT:
																																									snake_case_					:Optional[int]		=     False
																																			snake_case_					:Tuple		=     []

																							snake_case_					:Union[str, Any]		=     """ """.join(line.split()[1:]		).strip(""" \n"""		)
																	else:
																							current_lines.append(snake_case		)
											else:
																	if current_sub_level != "":
																							if current_sub_level in self.content:
																													self.parsing_error_list.append(
																													    f"""Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections."""		)
																							snake_case_					:Dict		=     Section(snake_case		,	self.level + """#"""		,	snake_case		)
																	else:
																							if current_lines != []:
																													self.text += "".join(snake_case		).strip()
																													if self.text != "" and self.text not in FILLER_TEXT:
																																			snake_case_					:Optional[int]		=     False

											if self.level == "" and not suppress_parsing_errors:
																	if self.parsing_error_list != [] or self.parsing_warning_list != []:
																							snake_case_					:Optional[Any]		=     """\n""".join("""-\t""" + x for x in self.parsing_error_list + self.parsing_warning_list		)
																							snake_case_					:List[Any]		=     f"""The following issues were found while parsing the README at `{self.name}`:\n""" + errors
																							raise ValueError(snake_case		)






					def   lowerCAmelCase_							(			self:				List[Any]		,	snake_case:				dict		)  ->      ReadmeValidatorOutput:
											snake_case_					:Tuple		=     []
											snake_case_					:str		=     []
											if structure["allow_empty"] is False:
																	# If content is expected
																	if self.is_empty_text and self.content == {}:
																							# If no content is found, mention it in the error_list
																							error_list.append(f"""Expected some content in section `{self.name}` but it is empty."""		)

											if structure["allow_empty_text"] is False:
																	# If some text is expected
																	if self.is_empty_text:
																							# If no text is found, mention it in the error_list
																							error_list.append(
																							    f"""Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)."""		)
        # Subsections Validation
											if structure["subsections"] is not None:
																	# If subsections are expected
																	if self.content == {}:
																							# If no subsections are present
																							snake_case_					:List[str]		=     [subsection["""name"""] for subsection in structure["""subsections"""]]
																							# Mention the expected values in the error_list
																							error_list.append(
																							    f"""Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values]		)}. Found 'None'."""		)
																	else:
																							# If some subsections are present
																							snake_case_					:List[Any]		=     [subsection["""name"""] for subsection in structure["""subsections"""]]
																							snake_case_					:Union[str, Any]		=     False
																							for idx, name in enumerate(snake_case		):
																													if name not in self.content:
																																			# If the expected subsection is not present
																																			error_list.append(f"""Section `{self.name}` is missing subsection: `{name}`."""		)
																																			snake_case_					:Any		=     True
																													else:
																																			# If the subsection is present, validate subsection, return the result
																																			# and concat the errors from subsection to section error_list

																																			# Skip sublevel validation if current level is `###`
																																			if self.level == "###":
																																									continue
																																			else:
																																									snake_case_,			snake_case_,			snake_case_					:Dict		=     self.content[name].validate(
																																									    structure["""subsections"""][idx]		)
																																			error_list += subsec_error_list
																																			warning_list += subsec_warning_list

																							if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
																													for name in self.content:
																																			if name not in structure_names:
																																									# If an extra subsection is present
																																									warning_list.append(
																																									    f"""`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown."""		)
											if error_list:
																	# If there are errors, do not return the dictionary as it is invalid
																	return {}, error_list, warning_list
											else:
																	return self.to_dict(), error_list, warning_list






					def   lowerCAmelCase_							(			self:				List[str]		)  ->      dict:
											return {
											    "name": self.name,
											    "text": self.text,
											    "is_empty_text": self.is_empty_text,
											    "subsections": [value.to_dict() for value in self.content.values()],
											}

class 							lowerCamelCase    (     _lowerCAmelCase							):  # Level 0






					'''simple docstring'''





					def __init__(			self:				str		,	snake_case:				str		,	snake_case:				List[str]		,	snake_case:				dict = None		,	snake_case:				bool = False		)  ->      Tuple:
											super().__init__(name=snake_case		,	level=""""""		)  # Not using lines here as we need to use a child class parse
											snake_case_					:List[str]		=     structure
											snake_case_					:List[str]		=     -2
											snake_case_					:Tuple		=     0
											snake_case_					:Tuple		=     lines
											if self.lines is not None:
																	self.parse(suppress_parsing_errors=snake_case		)






					def   lowerCAmelCase_							(			self:				Dict		)  ->      Any:
											if self.structure is None:
																	snake_case_,			snake_case_,			snake_case_					:int		=     self._validate(snake_case		)
											else:
																	snake_case_,			snake_case_,			snake_case_					:Tuple		=     self._validate(self.structure		)
											if error_list != [] or warning_list != []:
																	snake_case_					:str		=     """\n""".join(["""-\t""" + x for x in error_list + warning_list]		)
																	snake_case_					:Any		=     f"""The following issues were found for the README at `{self.name}`:\n""" + errors
																	raise ValueError(snake_case		)






					@classmethod
					def   lowerCAmelCase_							(			cls:				List[Any]		,	snake_case:				Path		,	snake_case:				dict = None		,	snake_case:				bool = False		)  ->      Any:
											with open(snake_case		,	encoding="""utf-8"""		) as f:
																	snake_case_					:List[Any]		=     f.readlines()
											return cls(snake_case		,	snake_case		,	snake_case		,	suppress_parsing_errors=snake_case		)






					@classmethod
					def   lowerCAmelCase_							(			cls:				int		,	snake_case:				str		,	snake_case:				dict = None		,	snake_case:				str = "root"		,	snake_case:				bool = False		)  ->      Dict:
											snake_case_					:List[Any]		=     string.split("""\n"""		)
											return cls(snake_case		,	snake_case		,	snake_case		,	suppress_parsing_errors=snake_case		)






					def   lowerCAmelCase_							(			self:				Union[str, Any]		,	snake_case:				bool = False		)  ->      Optional[int]:
											# Skip Tags
											snake_case_					:Optional[Any]		=     0

											for line in self.lines:
																	self.yaml_tags_line_count += 1
																	if line.strip(""" \n"""		) == "---":
																							self.tag_count += 1
																							if self.tag_count == 2:
																													break
																	line_count += 1
											if self.tag_count == 2:
																	snake_case_					:List[Any]		=     self.lines[line_count + 1 :]  # Get the last + 1 th item.
											else:
																	snake_case_					:Optional[int]		=     self.lines[self.tag_count :]
											super().parse(suppress_parsing_errors=snake_case		)






					def __str__(			self:				Any		)  ->      Tuple:
											return str(self.to_dict()		)






					def   lowerCAmelCase_							(			self:				Dict		,	snake_case:				List[Any]		)  ->      Tuple:
											snake_case_					:Dict		=     []
											snake_case_					:List[str]		=     []
											if self.yaml_tags_line_count == 0:
																	warning_list.append("""Empty YAML markers are present in the README."""		)
											elif self.tag_count == 0:
																	warning_list.append("""No YAML markers are present in the README."""		)
											elif self.tag_count == 1:
																	warning_list.append("""Only the start of YAML tags present in the README."""		)
											# Check how many first level sections are present.
											snake_case_					:Optional[Any]		=     len(self.content.keys()		)
											if num_first_level_keys > 1:
																	# If more than one, add to the error list, continue
																	error_list.append(
																	    f"""The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys()		)]		)}. Only one heading is expected. Skipping further validation for this README."""		)
											elif num_first_level_keys < 1:
																	# If less than one, append error.
																	error_list.append(
																	    """The README has no first-level headings. One heading is expected. Skipping further validation for this README."""		)

											else:
																	# If one exactly
																	snake_case_					:Any		=     list(self.content.keys()		)[0]  # Get the key
																	if start_key.startswith("""Dataset Card for"""		):  # Check correct start
																							# If the starting is correct, validate all the sections
																							snake_case_,			snake_case_,			snake_case_					:List[Any]		=     self.content[start_key].validate(
																							    readme_structure["""subsections"""][0]		)
																							error_list += sec_error_list
																							warning_list += sec_warning_list
																	else:
																							# If not found, append error
																							error_list.append(
																							    """No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."""		)
											if error_list:
																	# If there are errors, do not return the dictionary as it is invalid
																	return {}, error_list, warning_list
											else:
																	return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
						from argparse import ArgumentParser

						__a           =							ArgumentParser(usage="Validate the content (excluding YAML tags) of a README.md file.")
						ap.add_argument("readme_filepath")
						__a           =							ap.parse_args()
						__a           =							Path(args.readme_filepath)
						__a           =							ReadMe.from_readme(readme_filepath)







'''simple docstring'''







import glob
import os

from get_test_info import get_tester_classes


if __name__ == "__main__":
					snake_case_  :							Union[str, Any]            =	[]

					snake_case_  :							List[Any]            =	os.path.join("tests", "models", "**", "test_modeling_*.py")
					snake_case_  :							int            =	glob.glob(pattern)
					# TODO: deal with TF/Flax too
					snake_case_  :							Optional[int]            =	[
					    x for x in test_files if not (x.startswith("test_modeling_tf_") or x.startswith("test_modeling_flax_"))
					]

					for test_file in test_files:
										snake_case_  :							Any            =	get_tester_classes(test_file)
										for tester_class in tester_classes:
															# A few tester classes don't have `parent` parameter in `__init__`.
															# TODO: deal this better
															try:
																				snake_case_  :							Tuple            =	tester_class(parent=None)
															except Exception:
																				continue
															if hasattr(tester, "get_config"):
																				snake_case_  :							Any            =	tester.get_config()
																				for k, v in config.to_dict().items():
																									if isinstance(v, int):
																														snake_case_  :							int            =	None
																														if k in ["vocab_size"]:
																																			snake_case_  :							Optional[Any]            =	1_00
																														elif k in ["max_position_embeddings"]:
																																			snake_case_  :							Any            =	1_28
																														elif k in ["hidden_size", "d_model"]:
																																			snake_case_  :							str            =	40
																														elif k == ["num_layers", "num_hidden_layers", "num_encoder_layers", "num_decoder_layers"]:
																																			snake_case_  :							str            =	5
																														if target is not None and v > target:
																																			failures.append(
																																			    f'''{tester_class.__name__} will produce a `config` of type `{config.__class__.__name__}`'''
																																			    f''' with config["{k}"] = {v} which is too large for testing! Set its value to be smaller'''
																																			    f''' than {target}.'''
																																			)

					if len(failures) > 0:
										raise Exception(f'''There were {len(failures)} failures:\n''' + "\n".join(failures))





