



"""simple docstring"""







import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import MarianConfig, is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
		import os

		# The slow tests are often failing with OOM error on GPU
		# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
		# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
		UpperCAmelCase:  List[str]			     =				"""platform"""

		import jax
		import jax.numpy as jnp

		from transformers import MarianTokenizer
		from transformers.models.marian.modeling_flax_marian import FlaxMarianModel, FlaxMarianMTModel, shift_tokens_right


def  __SCREAMING_SNAKE_CASE (  __UpperCAmelCase   ,				__UpperCAmelCase   ,				__UpperCAmelCase=None   ,				__UpperCAmelCase=None   ,				__UpperCAmelCase=None   ,				__UpperCAmelCase=None   ,				__UpperCAmelCase=None   ,				__UpperCAmelCase=None   ,				):
		if attention_mask is None:
				_lowercase				: Union[str, Any]     =   np.where(input_ids != config.pad_token_id   ,				1   ,				0      )
		if decoder_attention_mask is None:
				_lowercase				: Dict     =   np.where(decoder_input_ids != config.pad_token_id   ,				1   ,				0      )
		if head_mask is None:
				_lowercase				: Any     =   np.ones((config.encoder_layers, config.encoder_attention_heads)      )
		if decoder_head_mask is None:
				_lowercase				: Union[str, Any]     =   np.ones((config.decoder_layers, config.decoder_attention_heads)      )
		if cross_attn_head_mask is None:
				_lowercase				: Optional[int]     =   np.ones((config.decoder_layers, config.decoder_attention_heads)      )
		return {
		    "input_ids": input_ids,
		    "decoder_input_ids": decoder_input_ids,
		    "attention_mask": attention_mask,
		    "decoder_attention_mask": attention_mask,
		}





class   UpperCamelCase	:




	"""simple docstring"""






	def __init__(     self				,UpperCAmelCase_				,UpperCAmelCase_=13				,UpperCAmelCase_=7				,UpperCAmelCase_=True				,UpperCAmelCase_=False				,UpperCAmelCase_=99				,UpperCAmelCase_=16				,UpperCAmelCase_=2				,UpperCAmelCase_=4				,UpperCAmelCase_=4				,UpperCAmelCase_="gelu"				,UpperCAmelCase_=0.1				,UpperCAmelCase_=0.1				,UpperCAmelCase_=32				,UpperCAmelCase_=2				,UpperCAmelCase_=1				,UpperCAmelCase_=0				,UpperCAmelCase_=0.02				,):
			_lowercase				: List[Any]     =   parent
			_lowercase				: str     =   batch_size
			_lowercase				: Tuple     =   seq_length
			_lowercase				: Optional[Any]     =   is_training
			_lowercase				: Optional[Any]     =   use_labels
			_lowercase				: Optional[int]     =   vocab_size
			_lowercase				: List[Any]     =   hidden_size
			_lowercase				: Dict     =   num_hidden_layers
			_lowercase				: Optional[Any]     =   num_attention_heads
			_lowercase				: Optional[Any]     =   intermediate_size
			_lowercase				: Tuple     =   hidden_act
			_lowercase				: Tuple     =   hidden_dropout_prob
			_lowercase				: Tuple     =   attention_probs_dropout_prob
			_lowercase				: Union[str, Any]     =   max_position_embeddings
			_lowercase				: Any     =   eos_token_id
			_lowercase				: Optional[Any]     =   pad_token_id
			_lowercase				: Dict     =   bos_token_id
			_lowercase				: Any     =   initializer_range






	def    lowerCamelCase__					(     self   ):
			_lowercase				: str     =   np.clip(ids_tensor([self.batch_size, self.seq_length - 1]				,self.vocab_size   )				,3				,self.vocab_size   )
			_lowercase				: Optional[int]     =   np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)				,dtype=np.intaa   ))				,-1   )

			_lowercase				: List[str]     =   shift_tokens_right(UpperCAmelCase_				,1				,2   )

			_lowercase				: Union[str, Any]     =   MarianConfig(
			    vocab_size=self.vocab_size				,d_model=self.hidden_size				,encoder_layers=self.num_hidden_layers				,decoder_layers=self.num_hidden_layers				,encoder_attention_heads=self.num_attention_heads				,decoder_attention_heads=self.num_attention_heads				,encoder_ffn_dim=self.intermediate_size				,decoder_ffn_dim=self.intermediate_size				,dropout=self.hidden_dropout_prob				,attention_dropout=self.attention_probs_dropout_prob				,max_position_embeddings=self.max_position_embeddings				,eos_token_id=self.eos_token_id				,bos_token_id=self.bos_token_id				,pad_token_id=self.pad_token_id				,initializer_range=self.initializer_range				,use_cache=UpperCAmelCase_				,)
			_lowercase				: Union[str, Any]     =   prepare_marian_inputs_dict(UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_   )
			return config, inputs_dict






	def    lowerCamelCase__					(     self   ):
			_lowercase     ,  _lowercase				: List[Any]     =   self.prepare_config_and_inputs()
			return config, inputs_dict






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_   ):
			_lowercase				: Dict     =   20
			_lowercase				: Dict     =   model_class_name(UpperCAmelCase_   )

			_lowercase				: str     =   model.encode(inputs_dict["""input_ids"""]   )

			_lowercase     ,  _lowercase				: int     =   (
			    inputs_dict["""decoder_input_ids"""],
			    inputs_dict["""decoder_attention_mask"""],
			)

			_lowercase				: List[Any]     =   model.init_cache(decoder_input_ids.shape[0]				,UpperCAmelCase_				,UpperCAmelCase_   )
			_lowercase				: Tuple     =   jnp.ones((decoder_input_ids.shape[0], max_decoder_length)				,dtype="""i4"""   )

			_lowercase				: Union[str, Any]     =   jnp.broadcast_to(
			    jnp.arange(decoder_input_ids.shape[-1] - 1   )[None, :]				,(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)				,)
			_lowercase				: str     =   model.decode(
			    decoder_input_ids[:, :-1]				,UpperCAmelCase_				,decoder_attention_mask=UpperCAmelCase_				,past_key_values=UpperCAmelCase_				,decoder_position_ids=UpperCAmelCase_				,)

			_lowercase				: Tuple     =   jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]				,dtype="""i4"""   )
			_lowercase				: Tuple     =   model.decode(
			    decoder_input_ids[:, -1:]				,UpperCAmelCase_				,decoder_attention_mask=UpperCAmelCase_				,past_key_values=outputs_cache.past_key_values				,decoder_position_ids=UpperCAmelCase_				,)

			_lowercase				: Optional[int]     =   model.decode(UpperCAmelCase_				,UpperCAmelCase_   )

			_lowercase				: Tuple     =   np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])   )   )
			self.parent.assertTrue(diff < 1E-3				,msg=f"""Max diff is {diff}"""   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_   ):
			_lowercase				: int     =   20
			_lowercase				: Tuple     =   model_class_name(UpperCAmelCase_   )

			_lowercase				: Tuple     =   model.encode(inputs_dict["""input_ids"""]   )

			_lowercase     ,  _lowercase				: Union[str, Any]     =   (
			    inputs_dict["""decoder_input_ids"""],
			    inputs_dict["""decoder_attention_mask"""],
			)

			_lowercase				: Tuple     =   jnp.concatenate(
			    [
			        decoder_attention_mask,
			        jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])   ),
			    ]				,axis=-1				,)

			_lowercase				: List[Any]     =   model.init_cache(decoder_input_ids.shape[0]				,UpperCAmelCase_				,UpperCAmelCase_   )
			_lowercase				: Tuple     =   jnp.broadcast_to(
			    jnp.arange(decoder_input_ids.shape[-1] - 1   )[None, :]				,(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)				,)

			_lowercase				: Dict     =   model.decode(
			    decoder_input_ids[:, :-1]				,UpperCAmelCase_				,decoder_attention_mask=UpperCAmelCase_				,past_key_values=UpperCAmelCase_				,decoder_position_ids=UpperCAmelCase_				,)
			_lowercase				: List[Any]     =   jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]				,dtype="""i4"""   )
			_lowercase				: List[Any]     =   model.decode(
			    decoder_input_ids[:, -1:]				,UpperCAmelCase_				,past_key_values=outputs_cache.past_key_values				,decoder_attention_mask=UpperCAmelCase_				,decoder_position_ids=UpperCAmelCase_				,)

			_lowercase				: Tuple     =   model.decode(UpperCAmelCase_				,UpperCAmelCase_				,decoder_attention_mask=UpperCAmelCase_   )

			_lowercase				: Optional[Any]     =   np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])   )   )
			self.parent.assertTrue(diff < 1E-3				,msg=f"""Max diff is {diff}"""   )







@require_flax
class   UpperCamelCase	(     snake_case					,   unittest.TestCase					,   snake_case       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       Tuple						       =  True
	SCREAMING_SNAKE_CASE_							:       Optional[Any]						       =  (FlaxMarianModel, FlaxMarianMTModel) if is_flax_available() else ()
	SCREAMING_SNAKE_CASE_							:       str						       =  (FlaxMarianMTModel,) if is_flax_available() else ()






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Any     =   FlaxMarianModelTester(self   )






	def    lowerCamelCase__					(     self   ):
			_lowercase     ,  _lowercase				: Optional[Any]     =   self.model_tester.prepare_config_and_inputs()
			for model_class in self.all_model_classes:
					self.model_tester.check_use_cache_forward(UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			_lowercase     ,  _lowercase				: str     =   self.model_tester.prepare_config_and_inputs()
			for model_class in self.all_model_classes:
					self.model_tester.check_use_cache_forward_with_attn_mask(UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			_lowercase     ,  _lowercase				: int     =   self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					with self.subTest(model_class.__name__   ):
							_lowercase				: Optional[int]     =   self._prepare_for_class(UpperCAmelCase_				,UpperCAmelCase_   )
							_lowercase				: Optional[int]     =   model_class(UpperCAmelCase_   )

							@jax.jit
							def encode_jitted(UpperCAmelCase_				,UpperCAmelCase_=None				,**UpperCAmelCase_   ):
									return model.encode(input_ids=UpperCAmelCase_				,attention_mask=UpperCAmelCase_   )

							with self.subTest("""JIT Enabled"""   ):
									_lowercase				: List[Any]     =   encode_jitted(**UpperCAmelCase_   ).to_tuple()

							with self.subTest("""JIT Disabled"""   ):
									with jax.disable_jit():
											_lowercase				: Union[str, Any]     =   encode_jitted(**UpperCAmelCase_   ).to_tuple()

							self.assertEqual(len(UpperCAmelCase_   )				,len(UpperCAmelCase_   )   )
							for jitted_output, output in zip(UpperCAmelCase_				,UpperCAmelCase_   ):
									self.assertEqual(jitted_output.shape				,output.shape   )






	def    lowerCamelCase__					(     self   ):
			_lowercase     ,  _lowercase				: int     =   self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					with self.subTest(model_class.__name__   ):
							_lowercase				: Any     =   model_class(UpperCAmelCase_   )
							_lowercase				: List[str]     =   model.encode(inputs_dict["""input_ids"""]				,inputs_dict["""attention_mask"""]   )

							_lowercase				: str     =   {
							    """decoder_input_ids""": inputs_dict["""decoder_input_ids"""],
							    """decoder_attention_mask""": inputs_dict["""decoder_attention_mask"""],
							    """encoder_outputs""": encoder_outputs,
							}

							@jax.jit
							def decode_jitted(UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_   ):
									return model.decode(
									    decoder_input_ids=UpperCAmelCase_				,decoder_attention_mask=UpperCAmelCase_				,encoder_outputs=UpperCAmelCase_				,)

							with self.subTest("""JIT Enabled"""   ):
									_lowercase				: List[Any]     =   decode_jitted(**UpperCAmelCase_   ).to_tuple()

							with self.subTest("""JIT Disabled"""   ):
									with jax.disable_jit():
											_lowercase				: Optional[Any]     =   decode_jitted(**UpperCAmelCase_   ).to_tuple()

							self.assertEqual(len(UpperCAmelCase_   )				,len(UpperCAmelCase_   )   )
							for jitted_output, output in zip(UpperCAmelCase_				,UpperCAmelCase_   ):
									self.assertEqual(jitted_output.shape				,output.shape   )






	@slow
	def    lowerCamelCase__					(     self   ):
			for model_class_name in self.all_model_classes:
					_lowercase				: List[Any]     =   model_class_name.from_pretrained("""Helsinki-NLP/opus-mt-en-de"""   )
					# FlaxMarianForSequenceClassification expects eos token in input_ids
					_lowercase				: Union[str, Any]     =   np.ones((1, 1)   ) * model.config.eos_token_id
					_lowercase				: Dict     =   model(UpperCAmelCase_   )
					self.assertIsNotNone(UpperCAmelCase_   )







@require_flax
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase	(     unittest.TestCase       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       Union[str, Any]						       =  None
	SCREAMING_SNAKE_CASE_							:       Optional[Any]						       =  None






	@classmethod
	def    lowerCamelCase__					(     cls   ):
			_lowercase				: str     =   f"""Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"""
			return cls






	@cached_property
	def    lowerCamelCase__					(     self   ):
			return MarianTokenizer.from_pretrained(self.model_name   )






	@property
	def    lowerCamelCase__					(     self   ):
			return self.tokenizer.eos_token_id






	@cached_property
	def    lowerCamelCase__					(     self   ):
			_lowercase				: FlaxMarianMTModel     =   FlaxMarianMTModel.from_pretrained(self.model_name   )
			self.assertEqual(model.config.decoder_start_token_id				,model.config.pad_token_id   )
			return model






	def    lowerCamelCase__					(     self				,**UpperCAmelCase_   ):
			_lowercase				: int     =   self.translate_src_text(**UpperCAmelCase_   )
			self.assertListEqual(self.expected_text				,UpperCAmelCase_   )






	def    lowerCamelCase__					(     self				,**UpperCAmelCase_   ):
			_lowercase				: Dict     =   self.tokenizer(self.src_text				,padding=UpperCAmelCase_				,return_tensors="""np"""				,**UpperCAmelCase_   )
			_lowercase				: List[Any]     =   self.model.generate(
			    model_inputs.input_ids				,attention_mask=model_inputs.attention_mask				,num_beams=2				,max_length=1_28				,).sequences
			_lowercase				: int     =   self.tokenizer.batch_decode(UpperCAmelCase_				,skip_special_tokens=UpperCAmelCase_   )
			return generated_words







@require_flax
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase	(     snake_case       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       Optional[int]						       =  "en"
	SCREAMING_SNAKE_CASE_							:       Any						       =  "fr"
	SCREAMING_SNAKE_CASE_							:       Dict						       =  [
	    "I am a small frog.",
	    "Now I can forget the 100 words of german that I know.",
	]
	SCREAMING_SNAKE_CASE_							:       Optional[Any]						       =  [
	    "Je suis une petite grenouille.",
	    "Maintenant, je peux oublier les 100 mots d'allemand que je connais.",
	]






	@slow
	def    lowerCamelCase__					(     self   ):
			self._assert_generated_batch_equal_expected()







@require_flax
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase	(     snake_case       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       str						       =  "fr"
	SCREAMING_SNAKE_CASE_							:       Union[str, Any]						       =  "en"
	SCREAMING_SNAKE_CASE_							:       Any						       =  [
	    "Donnez moi le micro.",
	    "Tom et Mary étaient assis à une table.",  # Accents
	]
	SCREAMING_SNAKE_CASE_							:       Union[str, Any]						       =  [
	    "Give me the microphone.",
	    "Tom and Mary were sitting at a table.",
	]






	@slow
	def    lowerCamelCase__					(     self   ):
			self._assert_generated_batch_equal_expected()







@require_flax
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase	(     snake_case       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       Optional[Any]						       =  "mt"
	SCREAMING_SNAKE_CASE_							:       Optional[Any]						       =  "en"
	SCREAMING_SNAKE_CASE_							:       List[Any]						       =  ["Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."]
	SCREAMING_SNAKE_CASE_							:       Any						       =  ["Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."]






	@slow
	def    lowerCamelCase__					(     self   ):
			self._assert_generated_batch_equal_expected()







@require_flax
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase	(     snake_case       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       Any						       =  "en"
	SCREAMING_SNAKE_CASE_							:       Tuple						       =  "de"
	SCREAMING_SNAKE_CASE_							:       Tuple						       =  [
	    "I am a small frog.",
	    "Now I can forget the 100 words of german that I know.",
	    "Tom asked his teacher for advice.",
	    "That's how I would do it.",
	    "Tom really admired Mary's courage.",
	    "Turn around and close your eyes.",
	]
	SCREAMING_SNAKE_CASE_							:       Optional[int]						       =  [
	    "Ich bin ein kleiner Frosch.",
	    "Jetzt kann ich die 100 Wörter des Deutschen vergessen, die ich kenne.",
	    "Tom bat seinen Lehrer um Rat.",
	    "So würde ich das machen.",
	    "Tom bewunderte Marias Mut wirklich.",
	    "Drehen Sie sich um und schließen Sie die Augen.",
	]






	@slow
	def    lowerCamelCase__					(     self   ):
			self._assert_generated_batch_equal_expected()







@require_flax
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase	(     snake_case       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       Union[str, Any]						       =  "en"
	SCREAMING_SNAKE_CASE_							:       Optional[int]						       =  "zh"
	SCREAMING_SNAKE_CASE_							:       Any						       =  ["My name is Wolfgang and I live in Berlin"]
	SCREAMING_SNAKE_CASE_							:       Any						       =  ["我叫沃尔夫冈 我住在柏林"]






	@slow
	def    lowerCamelCase__					(     self   ):
			self._assert_generated_batch_equal_expected()







@require_flax
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase	(     snake_case       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       Dict						       =  "ru"
	SCREAMING_SNAKE_CASE_							:       Optional[int]						       =  "fr"
	SCREAMING_SNAKE_CASE_							:       Optional[Any]						       =  ["Он показал мне рукопись своей новой пьесы."]
	SCREAMING_SNAKE_CASE_							:       Union[str, Any]						       =  ["Il m'a montré le manuscrit de sa nouvelle pièce."]






	@slow
	def    lowerCamelCase__					(     self   ):
			self._assert_generated_batch_equal_expected()







@require_flax
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase	(     snake_case       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       Dict						       =  "en"
	SCREAMING_SNAKE_CASE_							:       List[str]						       =  "ROMANCE"
	SCREAMING_SNAKE_CASE_							:       List[Any]						       =  [
	    ">>fr<< Don't spend so much time watching TV.",
	    ">>pt<< Your message has been sent.",
	    ">>es<< He's two years older than me.",
	]
	SCREAMING_SNAKE_CASE_							:       int						       =  [
	    "Ne passez pas autant de temps à regarder la télé.",
	    "A sua mensagem foi enviada.",
	    "Es dos años más viejo que yo.",
	]






	@slow
	def    lowerCamelCase__					(     self   ):
			self._assert_generated_batch_equal_expected()








import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
						import importlib.resources as pkg_resources
except ImportError:
						# Try backported to PY<37 `importlib_resources`.
						import importlib_resources as pkg_resources

from . import resources


UpperCAmelCase_			      =  '''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
UpperCAmelCase_			      =  f"""{BASE_REF_URL}/{__file__}"""
UpperCAmelCase_			      =  logging.getLogger(__name__)
def   UpperCAmelCase							(		A__     )							->					Tuple[Any, str]:
	_snake_case					:							Optional[int]					     =   pkg_resources.read_text(A__				,  A__     )
	return yaml.safe_load(A__     ), f'''{BASE_REF_URL}/resources/{resource}'''


UpperCAmelCase_      ,				UpperCAmelCase_			      =  load_yaml_resource('''readme_structure.yaml''')

UpperCAmelCase_			      =  [
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
UpperCAmelCase_			      =  Tuple[dict, List[str], List[str]]





class 		__SCREAMING_SNAKE_CASE      :






				"""simple docstring"""

				def __init__(				self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None						,	SCREAMING_SNAKE_CASE__ = False      ):






					"""simple docstring"""

					_snake_case					:							Optional[Any]					     =   name
					_snake_case					:							str					     =   level
					_snake_case					:							Optional[Any]					     =   lines
					_snake_case					:							Any					     =   """"""
					_snake_case					:							int					     =   True
					_snake_case					:							Dict					     =   {}
					_snake_case					:							List[str]					     =   []
					_snake_case					:							Optional[Any]					     =   []
					if self.lines is not None:
						self.parse(suppress_parsing_errors=SCREAMING_SNAKE_CASE__      )







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__ = False      ):






					"""simple docstring"""

					_snake_case					:							Dict					     =   """"""
					_snake_case					:							Dict					     =   []
					_snake_case					:							str					     =   False
					for line in self.lines:
						if line.strip(""" \n"""      ) == "":
							continue
						elif line.strip(""" \n"""      )[:3] == "```":
							_snake_case					:							Tuple					     =   not code_start
						elif line.split()[0] == self.level + "#" and not code_start:
							if current_sub_level != "":
								_snake_case					:							int					     =   Section(SCREAMING_SNAKE_CASE__						,	self.level + """#"""						,	SCREAMING_SNAKE_CASE__      )
								_snake_case					:							Optional[int]					     =   []
							else:
								if current_lines != []:
									self.text += "".join(SCREAMING_SNAKE_CASE__      ).strip()
									if self.text != "" and self.text not in FILLER_TEXT:
										_snake_case					:							Any					     =   False
									_snake_case					:							Tuple					     =   []

							_snake_case					:							Optional[Any]					     =   """ """.join(line.split()[1:]      ).strip(""" \n"""      )
						else:
							current_lines.append(SCREAMING_SNAKE_CASE__      )
					else:
						if current_sub_level != "":
							if current_sub_level in self.content:
								self.parsing_error_list.append(
								    f'''Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.'''      )
							_snake_case					:							Union[str, Any]					     =   Section(SCREAMING_SNAKE_CASE__						,	self.level + """#"""						,	SCREAMING_SNAKE_CASE__      )
						else:
							if current_lines != []:
								self.text += "".join(SCREAMING_SNAKE_CASE__      ).strip()
								if self.text != "" and self.text not in FILLER_TEXT:
									_snake_case					:							Dict					     =   False

					if self.level == "" and not suppress_parsing_errors:
						if self.parsing_error_list != [] or self.parsing_warning_list != []:
							_snake_case					:							List[Any]					     =   """\n""".join("""-\t""" + x for x in self.parsing_error_list + self.parsing_warning_list      )
							_snake_case					:							Optional[Any]					     =   f'''The following issues were found while parsing the README at `{self.name}`:\n''' + errors
							raise ValueError(SCREAMING_SNAKE_CASE__      )







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__      ):






					"""simple docstring"""

					_snake_case					:							Union[str, Any]					     =   []
					_snake_case					:							Tuple					     =   []
					if structure["allow_empty"] is False:
						# If content is expected
						if self.is_empty_text and self.content == {}:
							# If no content is found, mention it in the error_list
							error_list.append(f'''Expected some content in section `{self.name}` but it is empty.'''      )

					if structure["allow_empty_text"] is False:
						# If some text is expected
						if self.is_empty_text:
							# If no text is found, mention it in the error_list
							error_list.append(
							    f'''Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).'''      )
        # Subsections Validation
					if structure["subsections"] is not None:
						# If subsections are expected
						if self.content == {}:
							# If no subsections are present
							_snake_case					:							Optional[int]					     =   [subsection["""name"""] for subsection in structure["""subsections"""]]
							# Mention the expected values in the error_list
							error_list.append(
							    f'''Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]      )}. Found \'None\'.'''      )
						else:
							# If some subsections are present
							_snake_case					:							Tuple					     =   [subsection["""name"""] for subsection in structure["""subsections"""]]
							_snake_case					:							Dict					     =   False
							for idx, name in enumerate(SCREAMING_SNAKE_CASE__      ):
								if name not in self.content:
									# If the expected subsection is not present
									error_list.append(f'''Section `{self.name}` is missing subsection: `{name}`.'''      )
									_snake_case					:							int					     =   True
								else:
									# If the subsection is present, validate subsection, return the result
									# and concat the errors from subsection to section error_list

									# Skip sublevel validation if current level is `###`
									if self.level == "###":
										continue
									else:
										_snake_case     , _snake_case     , _snake_case					:							Tuple					     =   self.content[name].validate(
										    structure["""subsections"""][idx]      )
									error_list += subsec_error_list
									warning_list += subsec_warning_list

							if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
								for name in self.content:
									if name not in structure_names:
										# If an extra subsection is present
										warning_list.append(
										    f'''`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.'''      )
					if error_list:
						# If there are errors, do not return the dictionary as it is invalid
						return {}, error_list, warning_list
					else:
						return self.to_dict(), error_list, warning_list







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					return {
					    "name": self.name,
					    "text": self.text,
					    "is_empty_text": self.is_empty_text,
					    "subsections": [value.to_dict() for value in self.content.values()],
					}





class 		__SCREAMING_SNAKE_CASE      (				lowercase__      ):  # Level 0






				"""simple docstring"""

				def __init__(				self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None						,	SCREAMING_SNAKE_CASE__ = False      ):






					"""simple docstring"""

					super().__init__(name=SCREAMING_SNAKE_CASE__						,	level=""""""      )  # Not using lines here as we need to use a child class parse
					_snake_case					:							Optional[int]					     =   structure
					_snake_case					:							str					     =   -2
					_snake_case					:							Any					     =   0
					_snake_case					:							int					     =   lines
					if self.lines is not None:
						self.parse(suppress_parsing_errors=SCREAMING_SNAKE_CASE__      )







				def 		__lowerCamelCase(				self      ):






					"""simple docstring"""

					if self.structure is None:
						_snake_case     , _snake_case     , _snake_case					:							List[Any]					     =   self._validate(SCREAMING_SNAKE_CASE__      )
					else:
						_snake_case     , _snake_case     , _snake_case					:							str					     =   self._validate(self.structure      )
					if error_list != [] or warning_list != []:
						_snake_case					:							List[Any]					     =   """\n""".join(["""-\t""" + x for x in error_list + warning_list]      )
						_snake_case					:							Any					     =   f'''The following issues were found for the README at `{self.name}`:\n''' + errors
						raise ValueError(SCREAMING_SNAKE_CASE__      )







				@classmethod
				def 		__lowerCamelCase(				cls						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None						,	SCREAMING_SNAKE_CASE__ = False      ):






					"""simple docstring"""

					with open(SCREAMING_SNAKE_CASE__						,	encoding="""utf-8"""      ) as f:
						_snake_case					:							List[str]					     =   f.readlines()
					return cls(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	suppress_parsing_errors=SCREAMING_SNAKE_CASE__      )







				@classmethod
				def 		__lowerCamelCase(				cls						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None						,	SCREAMING_SNAKE_CASE__ = "root"						,	SCREAMING_SNAKE_CASE__ = False      ):






					"""simple docstring"""

					_snake_case					:							Tuple					     =   string.split("""\n"""      )
					return cls(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	suppress_parsing_errors=SCREAMING_SNAKE_CASE__      )







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__ = False      ):






					"""simple docstring"""

					_snake_case					:							Optional[int]					     =   0

					for line in self.lines:
						self.yaml_tags_line_count += 1
						if line.strip(""" \n"""      ) == "---":
							self.tag_count += 1
							if self.tag_count == 2:
								break
						line_count += 1
					if self.tag_count == 2:
						_snake_case					:							Optional[int]					     =   self.lines[line_count + 1 :]  # Get the last + 1 th item.
					else:
						_snake_case					:							List[str]					     =   self.lines[self.tag_count :]
					super().parse(suppress_parsing_errors=SCREAMING_SNAKE_CASE__      )







				def __str__(				self      ):






					"""simple docstring"""

					return str(self.to_dict()      )







				def 		__lowerCamelCase(				self						,	SCREAMING_SNAKE_CASE__      ):






					"""simple docstring"""

					_snake_case					:							str					     =   []
					_snake_case					:							str					     =   []
					if self.yaml_tags_line_count == 0:
						warning_list.append("""Empty YAML markers are present in the README."""      )
					elif self.tag_count == 0:
						warning_list.append("""No YAML markers are present in the README."""      )
					elif self.tag_count == 1:
						warning_list.append("""Only the start of YAML tags present in the README."""      )
					# Check how many first level sections are present.
					_snake_case					:							List[str]					     =   len(self.content.keys()      )
					if num_first_level_keys > 1:
						# If more than one, add to the error list, continue
						error_list.append(
						    f'''The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()      )]      )}. Only one heading is expected. Skipping further validation for this README.'''      )
					elif num_first_level_keys < 1:
						# If less than one, append error.
						error_list.append(
						    """The README has no first-level headings. One heading is expected. Skipping further validation for this README."""      )

					else:
						# If one exactly
						_snake_case					:							List[str]					     =   list(self.content.keys()      )[0]  # Get the key
						if start_key.startswith("""Dataset Card for"""      ):  # Check correct start
							# If the starting is correct, validate all the sections
							_snake_case     , _snake_case     , _snake_case					:							List[Any]					     =   self.content[start_key].validate(
							    readme_structure["""subsections"""][0]      )
							error_list += sec_error_list
							warning_list += sec_warning_list
						else:
							# If not found, append error
							error_list.append(
							    """No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."""      )
					if error_list:
						# If there are errors, do not return the dictionary as it is invalid
						return {}, error_list, warning_list
					else:
						return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
						from argparse import ArgumentParser

						UpperCAmelCase_			      =  ArgumentParser(usage='''Validate the content (excluding YAML tags) of a README.md file.''')
						ap.add_argument('''readme_filepath''')
						UpperCAmelCase_			      =  ap.parse_args()
						UpperCAmelCase_			      =  Path(args.readme_filepath)
						UpperCAmelCase_			      =  ReadMe.from_readme(readme_filepath)





