






import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


A										=      16
A										=      32



def        _lowerCamelCase(      lowerCAmelCase__					:		Optional[int]   ):





				'''simple docstring'''







				return int(x / 2**20   )


class 				__a				:





		'''simple docstring'''



		def __enter__(    self							):
						gc.collect()
						torch.cuda.empty_cache()
						torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
						SCREAMING_SNAKE_CASE_ :						Dict    				= torch.cuda.memory_allocated()
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= psutil.Process()

						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.cpu_mem_used()
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= True
						SCREAMING_SNAKE_CASE_ :						List[str]    				= threading.Thread(target=self.peak_monitor_func							)
						SCREAMING_SNAKE_CASE_ :						Tuple    				= True
						peak_monitor_thread.start()
						return self


		def 	__snake_case      (    self							):
						return self.process.memory_info().rss


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= -1

						while True:
										SCREAMING_SNAKE_CASE_ :						Tuple    				= max(self.cpu_mem_used()			,     self.cpu_peak							)

										# can't sleep or will not catch the peak right (this comment is here on purpose)
										# time.sleep(0.001) # 1msec

										if not self.peak_monitoring:
														break






		def __exit__(    self			,     *UpperCamelCase__							):
						SCREAMING_SNAKE_CASE_ :						Dict    				= False

						gc.collect()
						torch.cuda.empty_cache()
						SCREAMING_SNAKE_CASE_ :						Dict    				= torch.cuda.memory_allocated()
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= torch.cuda.max_memory_allocated()
						SCREAMING_SNAKE_CASE_ :						List[str]    				= bamb(self.end - self.begin							)
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= bamb(self.peak - self.begin							)

						SCREAMING_SNAKE_CASE_ :						Any    				= self.cpu_mem_used()
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= bamb(self.cpu_end - self.cpu_begin							)
						SCREAMING_SNAKE_CASE_ :						str    				= bamb(self.cpu_peak - self.cpu_begin							)
						# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
				from accelerate.test_utils.training import mocked_dataloaders

				A										=      mocked_dataloaders  # noqa: F811



def        _lowerCamelCase(      lowerCAmelCase__					:		Tuple     ,     lowerCAmelCase__					:		int   ):





				'''simple docstring'''







				if os.environ.get('TESTING_MOCKED_DATALOADERS'     ,     lowerCAmelCase__   ) == "1":
								SCREAMING_SNAKE_CASE_ :						Tuple    				= 2

				# New Code #
				# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
				SCREAMING_SNAKE_CASE_ :						List[str]    				= FullyShardedDataParallelPlugin(
				    state_dict_config=FullStateDictConfig(offload_to_cpu=lowerCAmelCase__     ,     ranka_only=lowerCAmelCase__   )     ,     optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=lowerCAmelCase__     ,     ranka_only=lowerCAmelCase__   )     ,     )

				# Initialize accelerator
				if args.with_tracking:
								SCREAMING_SNAKE_CASE_ :						List[str]    				= Accelerator(
								    cpu=args.cpu     ,     mixed_precision=args.mixed_precision     ,     log_with='wandb'     ,     project_dir=args.logging_dir     ,     fsdp_plugin=lowerCAmelCase__     ,     )
				else:
								SCREAMING_SNAKE_CASE_ :						Any    				= Accelerator(fsdp_plugin=lowerCAmelCase__   )
				accelerator.print(accelerator.distributed_type   )

				if hasattr(args.checkpointing_steps     ,     'isdigit'   ):
								if args.checkpointing_steps == "epoch":
												SCREAMING_SNAKE_CASE_ :						Dict    				= args.checkpointing_steps
								elif args.checkpointing_steps.isdigit():
												SCREAMING_SNAKE_CASE_ :						Any    				= int(args.checkpointing_steps   )
								else:
												raise ValueError(
												    F'''Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.'''   )
				else:
								SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= None
				# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
				SCREAMING_SNAKE_CASE_ :						int    				= config['lr']
				SCREAMING_SNAKE_CASE_ :						List[Any]    				= int(config['num_epochs']   )
				SCREAMING_SNAKE_CASE_ :						List[str]    				= int(config['seed']   )
				SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= int(config['batch_size']   )

				# We need to initialize the trackers we use, and also store our configuration
				if args.with_tracking:
								SCREAMING_SNAKE_CASE_ :						Tuple    				= vars(lowerCAmelCase__   )
								accelerator.init_trackers('fsdp_glue_no_trainer'     ,     lowerCAmelCase__   )

				SCREAMING_SNAKE_CASE_ :						int    				= AutoTokenizer.from_pretrained(args.model_name_or_path   )
				SCREAMING_SNAKE_CASE_ :						Any    				= load_dataset('glue'     ,     'mrpc'   )
				SCREAMING_SNAKE_CASE_ :						List[Any]    				= evaluate.load('glue'     ,     'mrpc'   )

				def tokenize_function(lowerCAmelCase__					:		Union[str, Any]   ):
								# max_length=None => use the model max length (it's actually the default)
								SCREAMING_SNAKE_CASE_ :						Any    				= tokenizer(examples['sentence1']     ,     examples['sentence2']     ,     truncation=lowerCAmelCase__     ,     max_length=lowerCAmelCase__   )
								return outputs

				# Apply the method we just defined to all the examples in all the splits of the dataset
				# starting with the main process first:
				with accelerator.main_process_first():
								SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= datasets.map(
								    lowerCAmelCase__     ,     batched=lowerCAmelCase__     ,     remove_columns=['idx', 'sentence1', 'sentence2']     ,     )

				# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
				# transformers library
				SCREAMING_SNAKE_CASE_ :						Dict    				= tokenized_datasets.rename_column('label'     ,     'labels'   )

				# If the batch size is too big we use gradient accumulation
				SCREAMING_SNAKE_CASE_ :						Tuple    				= 1
				if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
								SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= batch_size // MAX_GPU_BATCH_SIZE
								SCREAMING_SNAKE_CASE_ :						Optional[int]    				= MAX_GPU_BATCH_SIZE

				def collate_fn(lowerCAmelCase__					:		Union[str, Any]   ):
								# On TPU it's best to pad everything to the same length or training will be very slow.
								SCREAMING_SNAKE_CASE_ :						str    				= 128 if accelerator.distributed_type == DistributedType.TPU else None
								# When using mixed precision we want round multiples of 8/16
								if accelerator.mixed_precision == "fp8":
												SCREAMING_SNAKE_CASE_ :						Any    				= 16
								elif accelerator.mixed_precision != "no":
												SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= 8
								else:
												SCREAMING_SNAKE_CASE_ :						Optional[int]    				= None

								return tokenizer.pad(
								    lowerCAmelCase__     ,     padding='longest'     ,     max_length=lowerCAmelCase__     ,     pad_to_multiple_of=lowerCAmelCase__     ,     return_tensors='pt'     ,     )

				# Instantiate dataloaders.
				SCREAMING_SNAKE_CASE_ :						str    				= DataLoader(
				    tokenized_datasets['train']     ,     shuffle=lowerCAmelCase__     ,     collate_fn=lowerCAmelCase__     ,     batch_size=lowerCAmelCase__   )
				SCREAMING_SNAKE_CASE_ :						int    				= DataLoader(
				    tokenized_datasets['validation']     ,     shuffle=lowerCAmelCase__     ,     collate_fn=lowerCAmelCase__     ,     batch_size=lowerCAmelCase__   )

				set_seed(lowerCAmelCase__   )

				# Instantiate the model (we build the model here so that the seed also control new weights initialization)
				SCREAMING_SNAKE_CASE_ :						Optional[int]    				= AutoModelForSequenceClassification.from_pretrained(
				    args.model_name_or_path     ,     return_dict=lowerCAmelCase__     ,     low_cpu_mem_usage=lowerCAmelCase__   )

				# New Code #
				# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
				SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= accelerator.prepare(lowerCAmelCase__   )
				accelerator.print(lowerCAmelCase__   )

				# Instantiate optimizer
				# New Code #
				# For FSDP feature, at present it doesn't support multiple parameter groups,
				# so we need to create a single parameter group for the whole model
				SCREAMING_SNAKE_CASE_ :						List[str]    				= torch.optim.AdamW(params=model.parameters()     ,     lr=lowerCAmelCase__     ,     weight_decay=2E-4   )

				# Instantiate scheduler
				SCREAMING_SNAKE_CASE_ :						Tuple    				= get_linear_schedule_with_warmup(
				    optimizer=lowerCAmelCase__     ,     num_warmup_steps=10     ,     num_training_steps=(len(lowerCAmelCase__   ) * num_epochs) // gradient_accumulation_steps     ,     )

				# New Code #
				# For FSDP feature, prepare everything except the model as we have already prepared the model
				# before creating the optimizer
				# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
				# prepare method.
				SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_ :						Optional[int]    				= accelerator.prepare(
				    lowerCAmelCase__     ,     lowerCAmelCase__     ,     lowerCAmelCase__     ,     lowerCAmelCase__   )

				SCREAMING_SNAKE_CASE_ :						int    				= 0

				# Potentially load in the weights and states from a previous save
				if args.resume_from_checkpoint:
								if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
												accelerator.print(F'''Resumed from checkpoint: {args.resume_from_checkpoint}'''   )
												accelerator.load_state(args.resume_from_checkpoint   )
												SCREAMING_SNAKE_CASE_ :						List[str]    				= os.path.basename(args.resume_from_checkpoint   )
								else:
												# Get the most recent checkpoint
												SCREAMING_SNAKE_CASE_ :						int    				= [f.name for f in os.scandir(os.getcwd()   ) if f.is_dir()]
												dirs.sort(key=os.path.getctime   )
												SCREAMING_SNAKE_CASE_ :						Dict    				= dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
								# Extract `epoch_{i}` or `step_{i}`
								SCREAMING_SNAKE_CASE_ :						int    				= os.path.splitext(lowerCAmelCase__   )[0]

								if "epoch" in training_difference:
												num_epochs -= int(training_difference.replace('epoch_'     ,     ''   )   )
												SCREAMING_SNAKE_CASE_ :						List[str]    				= None
								else:
												SCREAMING_SNAKE_CASE_ :						Dict    				= int(training_difference.replace('step_'     ,     ''   )   )
												num_epochs -= resume_step // len(lowerCAmelCase__   )
												# If resuming by step, we also need to know exactly how far into the DataLoader we went
												SCREAMING_SNAKE_CASE_ :						Dict    				= (num_epochs * len(lowerCAmelCase__   )) - resume_step

    # Now we train the model
				for epoch in range(lowerCAmelCase__   ):
								# New Code #
								# context manager to track the peak memory usage during the training epoch
								with TorchTracemalloc() as tracemalloc:
												model.train()
												if args.with_tracking:
																SCREAMING_SNAKE_CASE_ :						List[Any]    				= 0
												for step, batch in enumerate(lowerCAmelCase__   ):
																# We need to skip steps until we reach the resumed step
																if args.resume_from_checkpoint and epoch == 0:
																				if resume_step is not None and step < resume_step:
																								pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
																batch.to(accelerator.device   )
																SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= model(**lowerCAmelCase__   )
																SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= outputs.loss
																SCREAMING_SNAKE_CASE_ :						str    				= loss / gradient_accumulation_steps
																# We keep track of the loss at each epoch
																if args.with_tracking:
																				total_loss += loss.detach().float()
																accelerator.backward(lowerCAmelCase__   )
																if step % gradient_accumulation_steps == 0:
																				optimizer.step()
																				lr_scheduler.step()
																				optimizer.zero_grad()
																				# accelerator.print(lr_scheduler.get_lr())

																overall_step += 1

																if isinstance(lowerCAmelCase__     ,     lowerCAmelCase__   ):
																				SCREAMING_SNAKE_CASE_ :						Optional[int]    				= F'''step_{overall_step}'''
																				if overall_step % checkpointing_steps == 0:
																								if args.output_dir is not None:
																												SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= os.path.join(args.output_dir     ,     lowerCAmelCase__   )
																								accelerator.save_state(lowerCAmelCase__   )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
								accelerator.print('Memory before entering the train : {}'.format(bamb(tracemalloc.begin   )   )   )
								accelerator.print('Memory consumed at the end of the train (end-begin): {}'.format(tracemalloc.used   )   )
								accelerator.print('Peak Memory consumed during the train (max-begin): {}'.format(tracemalloc.peaked   )   )
								accelerator.print(
								    'Total Peak Memory consumed during the train (max): {}'.format(
								        tracemalloc.peaked + bamb(tracemalloc.begin   )   )   )
								# Logging the peak memory usage of the GPU to the tracker
								if args.with_tracking:
												accelerator.log(
												    {
												        'train_total_peak_memory': tracemalloc.peaked + bamb(tracemalloc.begin   ),
												    }     ,     step=lowerCAmelCase__     ,     )

								# New Code #
								# context manager to track the peak memory usage during the evaluation
								with TorchTracemalloc() as tracemalloc:
												model.eval()
												for step, batch in enumerate(lowerCAmelCase__   ):
																# We could avoid this line since we set the accelerator with `device_placement=True`.
																batch.to(accelerator.device   )
																with torch.no_grad():
																				SCREAMING_SNAKE_CASE_ :						int    				= model(**lowerCAmelCase__   )
																SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= outputs.logits.argmax(dim=-1   )
																SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_ :						Dict    				= accelerator.gather_for_metrics((predictions, batch['labels'])   )
																metric.add_batch(
																    predictions=lowerCAmelCase__     ,     references=lowerCAmelCase__     ,     )

												SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= metric.compute()
												# Use accelerator.print to print only on the main process.
												accelerator.print(F'''epoch {epoch}:'''     ,     lowerCAmelCase__   )
												if args.with_tracking:
																accelerator.log(
																    {
																        'accuracy': eval_metric['accuracy'],
																        'f1': eval_metric['f1'],
																        'train_loss': total_loss.item() / len(lowerCAmelCase__   ),
																    }     ,     step=lowerCAmelCase__     ,     )

												if checkpointing_steps == "epoch":
																SCREAMING_SNAKE_CASE_ :						List[str]    				= F'''epoch_{epoch}'''
																if args.output_dir is not None:
																				SCREAMING_SNAKE_CASE_ :						str    				= os.path.join(args.output_dir     ,     lowerCAmelCase__   )
																accelerator.save_state(lowerCAmelCase__   )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
								accelerator.print('Memory before entering the eval : {}'.format(bamb(tracemalloc.begin   )   )   )
								accelerator.print('Memory consumed at the end of the eval (end-begin): {}'.format(tracemalloc.used   )   )
								accelerator.print('Peak Memory consumed during the eval (max-begin): {}'.format(tracemalloc.peaked   )   )
								accelerator.print(
								    'Total Peak Memory consumed during the eval (max): {}'.format(tracemalloc.peaked + bamb(tracemalloc.begin   )   )   )
								# Logging the peak memory usage of the GPU to the tracker
								if args.with_tracking:
												accelerator.log(
												    {
												        'eval_total_peak_memory': tracemalloc.peaked + bamb(tracemalloc.begin   ),
												    }     ,     step=lowerCAmelCase__     ,     )

				if args.with_tracking:
								accelerator.end_training()



def        _lowerCamelCase(      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= argparse.ArgumentParser(description='Simple example of training script.'   )
				parser.add_argument(
				    '--mixed_precision'     ,     type=lowerCAmelCase__     ,     default=lowerCAmelCase__     ,     choices=['no', 'fp16', 'bf16', 'fp8']     ,     help='Whether to use mixed precision. Choose'
				    'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
				    'and an Nvidia Ampere GPU.'     ,     )
				parser.add_argument('--cpu'     ,     action='store_true'     ,     help='If passed, will train on the CPU.'   )
				parser.add_argument(
				    '--checkpointing_steps'     ,     type=lowerCAmelCase__     ,     default=lowerCAmelCase__     ,     help='Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'     ,     )
				parser.add_argument(
				    '--resume_from_checkpoint'     ,     type=lowerCAmelCase__     ,     default=lowerCAmelCase__     ,     help='If the training should continue from a checkpoint folder.'     ,     )
				parser.add_argument(
				    '--with_tracking'     ,     action='store_true'     ,     help='Whether to load in all available experiment trackers from the environment and use them for logging.'     ,     )
				parser.add_argument(
				    '--output_dir'     ,     type=lowerCAmelCase__     ,     default='.'     ,     help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'     ,     )
				parser.add_argument(
				    '--logging_dir'     ,     type=lowerCAmelCase__     ,     default='logs'     ,     help='Location on where to store experiment tracking logs`'     ,     )
				parser.add_argument(
				    '--model_name_or_path'     ,     type=lowerCAmelCase__     ,     help='Path to pretrained model or model identifier from huggingface.co/models.'     ,     required=lowerCAmelCase__     ,     )
				SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= parser.parse_args()
				SCREAMING_SNAKE_CASE_ :						int    				= {'lr': 2E-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
				training_function(lowerCAmelCase__     ,     lowerCAmelCase__   )


if __name__ == "__main__":
				main()







import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MarkupLMForQuestionAnswering,
				    MarkupLMForSequenceClassification,
				    MarkupLMForTokenClassification,
				    MarkupLMModel,
				)

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer


class 				__a				:





		'''simple docstring'''



		def __init__(    self			,     UpperCamelCase__			,     UpperCamelCase__=13			,     UpperCamelCase__=7			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__=99			,     UpperCamelCase__=32			,     UpperCamelCase__=5			,     UpperCamelCase__=4			,     UpperCamelCase__=37			,     UpperCamelCase__="gelu"			,     UpperCamelCase__=0.1			,     UpperCamelCase__=0.1			,     UpperCamelCase__=512			,     UpperCamelCase__=16			,     UpperCamelCase__=2			,     UpperCamelCase__=0.02			,     UpperCamelCase__=3			,     UpperCamelCase__=None			,     UpperCamelCase__=20			,     UpperCamelCase__=30			,     UpperCamelCase__=2			,     UpperCamelCase__=2			,     UpperCamelCase__=10			,     ):
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= parent
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= batch_size
						SCREAMING_SNAKE_CASE_ :						int    				= seq_length
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= is_training
						SCREAMING_SNAKE_CASE_ :						Dict    				= use_input_mask
						SCREAMING_SNAKE_CASE_ :						Dict    				= use_token_type_ids
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= use_labels
						SCREAMING_SNAKE_CASE_ :						int    				= vocab_size
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= hidden_size
						SCREAMING_SNAKE_CASE_ :						Any    				= num_hidden_layers
						SCREAMING_SNAKE_CASE_ :						int    				= num_attention_heads
						SCREAMING_SNAKE_CASE_ :						Any    				= intermediate_size
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= hidden_act
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= hidden_dropout_prob
						SCREAMING_SNAKE_CASE_ :						List[str]    				= attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= max_position_embeddings
						SCREAMING_SNAKE_CASE_ :						Tuple    				= type_vocab_size
						SCREAMING_SNAKE_CASE_ :						str    				= type_sequence_label_size
						SCREAMING_SNAKE_CASE_ :						Dict    				= initializer_range
						SCREAMING_SNAKE_CASE_ :						int    				= num_labels
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= scope
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= max_xpath_tag_unit_embeddings
						SCREAMING_SNAKE_CASE_ :						Tuple    				= max_xpath_subs_unit_embeddings
						SCREAMING_SNAKE_CASE_ :						Dict    				= tag_pad_id
						SCREAMING_SNAKE_CASE_ :						str    				= subs_pad_id
						SCREAMING_SNAKE_CASE_ :						Tuple    				= max_depth


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= ids_tensor([self.batch_size, self.seq_length]			,     self.vocab_size							)

						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= ids_tensor(
						    [self.batch_size, self.seq_length, self.max_depth]			,     self.max_xpath_tag_unit_embeddings							)

						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= ids_tensor(
						    [self.batch_size, self.seq_length, self.max_depth]			,     self.max_xpath_subs_unit_embeddings							)

						SCREAMING_SNAKE_CASE_ :						Any    				= None
						if self.use_input_mask:
										SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= ids_tensor([self.batch_size, self.seq_length]			,     vocab_size=2							)

						SCREAMING_SNAKE_CASE_ :						Dict    				= None
						if self.use_token_type_ids:
										SCREAMING_SNAKE_CASE_ :						int    				= ids_tensor([self.batch_size, self.seq_length]			,     self.type_vocab_size							)

						SCREAMING_SNAKE_CASE_ :						int    				= None
						SCREAMING_SNAKE_CASE_ :						Tuple    				= None
						if self.use_labels:
										SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= ids_tensor([self.batch_size]			,     self.type_sequence_label_size							)
										SCREAMING_SNAKE_CASE_ :						Any    				= ids_tensor([self.batch_size, self.seq_length]			,     self.num_labels							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= self.get_config()

						return (
						    config,
						    input_ids,
						    xpath_tags_seq,
						    xpath_subs_seq,
						    token_type_ids,
						    input_mask,
						    sequence_labels,
						    token_labels,
						)


		def 	__snake_case      (    self							):
						return MarkupLMConfig(
						    vocab_size=self.vocab_size			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     max_position_embeddings=self.max_position_embeddings			,     type_vocab_size=self.type_vocab_size			,     initializer_range=self.initializer_range			,     max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings			,     max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings			,     tag_pad_id=self.tag_pad_id			,     subs_pad_id=self.subs_pad_id			,     max_depth=self.max_depth			,     )


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMModel(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						print('Configs:'			,     model.config.tag_pad_id			,     model.config.subs_pad_id							)
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= model(UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						List[str]    				= model(UpperCamelCase__			,     token_type_ids=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						int    				= model(UpperCamelCase__							)
						self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)							)
						self.parent.assertEqual(result.pooler_output.shape			,     (self.batch_size, self.hidden_size)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Dict    				= self.num_labels
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMForSequenceClassification(UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						str    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     labels=UpperCamelCase__			,     )
						self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.num_labels)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.num_labels
						SCREAMING_SNAKE_CASE_ :						List[str]    				= MarkupLMForTokenClassification(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						str    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     labels=UpperCamelCase__			,     )
						self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.seq_length, self.num_labels)							)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= MarkupLMForQuestionAnswering(config=UpperCamelCase__							)
						model.to(UpperCamelCase__							)
						model.eval()
						SCREAMING_SNAKE_CASE_ :						int    				= model(
						    UpperCamelCase__			,     xpath_tags_seq=UpperCamelCase__			,     xpath_subs_seq=UpperCamelCase__			,     attention_mask=UpperCamelCase__			,     token_type_ids=UpperCamelCase__			,     start_positions=UpperCamelCase__			,     end_positions=UpperCamelCase__			,     )
						self.parent.assertEqual(result.start_logits.shape			,     (self.batch_size, self.seq_length)							)
						self.parent.assertEqual(result.end_logits.shape			,     (self.batch_size, self.seq_length)							)






		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Any    				= self.prepare_config_and_inputs()
						(
						    (
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							(
						    SCREAMING_SNAKE_CASE_
						)				,							
						) :						int    				= config_and_inputs
						SCREAMING_SNAKE_CASE_ :						Any    				= {
						    'input_ids': input_ids,
						    'xpath_tags_seq': xpath_tags_seq,
						    'xpath_subs_seq': xpath_subs_seq,
						    'token_type_ids': token_type_ids,
						    'attention_mask': input_mask,
						}
						return config, inputs_dict






@require_torch
class 				__a				(	__A			,						__A			,						unittest.TestCase    ):





		'''simple docstring'''



		UpperCAmelCase__   :					Dict					     =    (
		    (
		        MarkupLMModel,
		        MarkupLMForSequenceClassification,
		        MarkupLMForTokenClassification,
		        MarkupLMForQuestionAnswering,
		    )
		    if is_torch_available()
		    else None
		)
		UpperCAmelCase__   :					Union[str, Any]					     =    (
		    {
		        """feature-extraction""": MarkupLMModel,
		        """question-answering""": MarkupLMForQuestionAnswering,
		        """text-classification""": MarkupLMForSequenceClassification,
		        """token-classification""": MarkupLMForTokenClassification,
		        """zero-shot""": MarkupLMForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__							):
						# ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]`
						# (batch of pretokenized examples).
						return True


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						int    				= MarkupLMModelTester(self							)
						SCREAMING_SNAKE_CASE_ :						Any    				= ConfigTester(self			,     config_class=UpperCamelCase__			,     hidden_size=37							)


		def 	__snake_case      (    self							):
						self.config_tester.run_common_tests()


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*UpperCamelCase__							)


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase__							)


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Any    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*UpperCamelCase__							)






		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*UpperCamelCase__							)






def        _lowerCamelCase(      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_ :						int    				= '\n    <!DOCTYPE html>\n    <html>\n    <head>\n    <title>Page Title</title>\n    </head>\n    <body>\n\n    <h1>This is a Heading</h1>\n    <p>This is a paragraph.</p>\n\n    </body>\n    </html>\n    '

				return html_string


@require_torch
class 				__a				(	unittest.TestCase    ):





		'''simple docstring'''



		@cached_property
		def 	__snake_case      (    self							):
						# TODO use from_pretrained here
						SCREAMING_SNAKE_CASE_ :						List[str]    				= MarkupLMFeatureExtractor()
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= MarkupLMTokenizer.from_pretrained('microsoft/markuplm-base'							)

						return MarkupLMProcessor(UpperCamelCase__			,     UpperCamelCase__							)






		@slow
		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Tuple    				= MarkupLMModel.from_pretrained('microsoft/markuplm-base'							).to(UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= self.default_processor

						SCREAMING_SNAKE_CASE_ :						int    				= processor(prepare_html_string()			,     return_tensors='pt'							)
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= inputs.to(UpperCamelCase__							)

						# forward pass
						with torch.no_grad():
										SCREAMING_SNAKE_CASE_ :						str    				= model(**UpperCamelCase__							)

						# verify the last hidden states
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= torch.Size([1, 14, 768]							)
						self.assertEqual(outputs.last_hidden_state.shape			,     UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Tuple    				= torch.tensor(
						    [[0.06_75, -0.00_52, 0.50_01], [-0.22_81, 0.08_02, 0.21_92], [-0.05_83, -0.33_11, 0.11_85]]							).to(UpperCamelCase__							)

						self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]			,     UpperCamelCase__			,     atol=1E-4							)							)