






"""simple docstring"""


import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing the checkpointing capability,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

lowercase_								=	16
lowercase_								=	32



def    lowerCAmelCase							(       __UpperCamelCase						,			__UpperCamelCase = 1_6					):




  """simple docstring"""







  __A		     =					AutoTokenizer.from_pretrained('''bert-base-cased'''					)
  __A		     =					load_dataset('''glue'''						,			'''mrpc'''					)

  def tokenize_function(__UpperCamelCase					):
    # max_length=None => use the model max length (it's actually the default)
    __A		     =					tokenizer(examples['''sentence1''']						,			examples['''sentence2''']						,			truncation=A_						,			max_length=A_					)
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  # starting with the main process first:
  with accelerator.main_process_first():
    __A		     =					datasets.map(
        A_						,			batched=A_						,			remove_columns=['''idx''', '''sentence1''', '''sentence2''']						,			)

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  __A		     =					tokenized_datasets.rename_column('''label'''						,			'''labels'''					)

  def collate_fn(__UpperCamelCase					):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    __A		     =					1_2_8 if accelerator.distributed_type == DistributedType.TPU else None
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
      __A		     =					1_6
    elif accelerator.mixed_precision != "no":
      __A		     =					8
    else:
      __A		     =					None

    return tokenizer.pad(
        A_						,			padding='''longest'''						,			max_length=A_						,			pad_to_multiple_of=A_						,			return_tensors='''pt'''						,			)

  # Instantiate dataloaders.
  __A		     =					DataLoader(
      tokenized_datasets['''train''']						,			shuffle=A_						,			collate_fn=A_						,			batch_size=A_					)
  __A		     =					DataLoader(
      tokenized_datasets['''validation''']						,			shuffle=A_						,			collate_fn=A_						,			batch_size=A_					)

  return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
       from accelerate.test_utils.training import mocked_dataloaders

       lowercase_								=	mocked_dataloaders  # noqa: F811



def    lowerCAmelCase							(       __UpperCamelCase						,			__UpperCamelCase					):




  """simple docstring"""







  if os.environ.get('''TESTING_MOCKED_DATALOADERS'''						,			A_					) == "1":
    __A		     =					2
  # Initialize accelerator
  __A		     =					Accelerator(cpu=args.cpu						,			mixed_precision=args.mixed_precision					)
  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  __A		     =					config['''lr''']
  __A		     =					int(config['''num_epochs''']					)
  __A		     =					int(config['''seed''']					)
  __A		     =					int(config['''batch_size''']					)

  # New Code #
  # Parse out whether we are saving every epoch or after a certain number of batches
  if hasattr(args.checkpointing_steps						,			'''isdigit'''					):
    if args.checkpointing_steps == "epoch":
      __A		     =					args.checkpointing_steps
    elif args.checkpointing_steps.isdigit():
      __A		     =					int(args.checkpointing_steps					)
    else:
      raise ValueError(
          f'Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.'					)
  else:
    __A		     =					None

  set_seed(A_					)

  __A		     =					get_dataloaders(A_						,			A_					)
  __A		     =					evaluate.load('''glue'''						,			'''mrpc'''					)

  # If the batch size is too big we use gradient accumulation
  __A		     =					1
  if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
    __A		     =					batch_size // MAX_GPU_BATCH_SIZE
    __A		     =					MAX_GPU_BATCH_SIZE

  # Instantiate the model (we build the model here so that the seed also control new weights initialization)
  __A		     =					AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''						,			return_dict=A_					)

  # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
  # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
  # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
  __A		     =					model.to(accelerator.device					)

  # Instantiate optimizer
  __A		     =					AdamW(params=model.parameters()						,			lr=A_					)

  # Instantiate scheduler
  __A		     =					get_linear_schedule_with_warmup(
      optimizer=A_						,			num_warmup_steps=1_0_0						,			num_training_steps=(len(A_					) * num_epochs) // gradient_accumulation_steps						,			)

  # Prepare everything
  # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  # prepare method.
  __A		     =					accelerator.prepare(
      A_						,			A_						,			A_						,			A_						,			A_					)

  # New Code #
  # We need to keep track of how many total steps we have iterated over
  __A		     =					0
  # We also need to keep track of the stating epoch so files are named properly
  __A		     =					0

  # We need to load the checkpoint back in before training here with `load_state`
  # The total number of epochs is adjusted based on where the state is being loaded from,
  # as we assume continuation of the same training script
  if args.resume_from_checkpoint:
    if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
      accelerator.print(f'Resumed from checkpoint: {args.resume_from_checkpoint}'					)
      accelerator.load_state(args.resume_from_checkpoint					)
      __A		     =					os.path.basename(args.resume_from_checkpoint					)
    else:
      # Get the most recent checkpoint
      __A		     =					[f.name for f in os.scandir(os.getcwd()					) if f.is_dir()]
      dirs.sort(key=os.path.getctime					)
      __A		     =					dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
    # Extract `epoch_{i}` or `step_{i}`
    __A		     =					os.path.splitext(A_					)[0]

    if "epoch" in training_difference:
      __A		     =					int(training_difference.replace('''epoch_'''						,			''''''					)					) + 1
      __A		     =					None
    else:
      __A		     =					int(training_difference.replace('''step_'''						,			''''''					)					)
      __A		     =					resume_step // len(A_					)
      resume_step -= starting_epoch * len(A_					)

    # Now we train the model
  for epoch in range(A_						,			A_					):
    model.train()
    # New Code #
    if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:
      # We need to skip steps until we reach the resumed step
      __A		     =					accelerator.skip_first_batches(A_						,			A_					)
      overall_step += resume_step
    else:
      # After the first iteration though, we need to go back to the original dataloader
      __A		     =					train_dataloader
    for step, batch in enumerate(A_					):
      # We could avoid this line since we set the accelerator with `device_placement=True`.
      batch.to(accelerator.device					)
      __A		     =					model(**A_					)
      __A		     =					outputs.loss
      __A		     =					loss / gradient_accumulation_steps
      accelerator.backward(A_					)
      if step % gradient_accumulation_steps == 0:
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
      # New Code #
      overall_step += 1

      # New Code #
      # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
      # These are saved to folders named `step_{overall_step}`
      # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
      # If mixed precision was used, will also save a "scalar.bin" file
      if isinstance(A_						,			A_					):
        __A		     =					f'step_{overall_step}'
        if overall_step % checkpointing_steps == 0:
          if args.output_dir is not None:
            __A		     =					os.path.join(args.output_dir						,			A_					)
          accelerator.save_state(A_					)

    model.eval()
    for step, batch in enumerate(A_					):
      # We could avoid this line since we set the accelerator with `device_placement=True` (the default).
      batch.to(accelerator.device					)
      with torch.no_grad():
        __A		     =					model(**A_					)
      __A		     =					outputs.logits.argmax(dim=-1					)
      __A		     =					accelerator.gather_for_metrics((predictions, batch['''labels'''])					)
      metric.add_batch(
          predictions=A_						,			references=A_						,			)

    __A		     =					metric.compute()
    # Use accelerator.print to print only on the main process.
    accelerator.print(f'epoch {epoch}:'						,			A_					)

    # New Code #
    # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
    # These are saved to folders named `epoch_{epoch}`
    # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
    # If mixed precision was used, will also save a "scalar.bin" file
    if checkpointing_steps == "epoch":
      __A		     =					f'epoch_{epoch}'
      if args.output_dir is not None:
        __A		     =					os.path.join(args.output_dir						,			A_					)
      accelerator.save_state(A_					)



def    lowerCAmelCase							(       ):




  """simple docstring"""







  __A		     =					argparse.ArgumentParser(description='''Simple example of training script.'''					)
  parser.add_argument(
      '''--mixed_precision'''						,			type=A_						,			default=A_						,			choices=['''no''', '''fp16''', '''bf16''', '''fp8''']						,			help='''Whether to use mixed precision. Choose'''
      '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
      '''and an Nvidia Ampere GPU.'''						,			)
  parser.add_argument('''--cpu'''						,			action='''store_true'''						,			help='''If passed, will train on the CPU.'''					)
  parser.add_argument(
      '''--checkpointing_steps'''						,			type=A_						,			default=A_						,			help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''						,			)
  parser.add_argument(
      '''--output_dir'''						,			type=A_						,			default='''.'''						,			help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''						,			)
  parser.add_argument(
      '''--resume_from_checkpoint'''						,			type=A_						,			default=A_						,			help='''If the training should continue from a checkpoint folder.'''						,			)
  __A		     =					parser.parse_args()
  __A		     =					{'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 4_2, '''batch_size''': 1_6}
  training_function(A_						,			A_					)


if __name__ == "__main__":
       main()






'''simple docstring'''





import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    MarkupLMForQuestionAnswering,
						    MarkupLMForSequenceClassification,
						    MarkupLMForTokenClassification,
						    MarkupLMModel,
						)

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer

class     a :




				def __init__(				self						:					Optional[Any],     SCREAMING_SNAKE_CASE_						:					Dict,     SCREAMING_SNAKE_CASE_						:					Any=13,     SCREAMING_SNAKE_CASE_						:					Any=7,     SCREAMING_SNAKE_CASE_						:					Union[str, Any]=True,     SCREAMING_SNAKE_CASE_						:					List[str]=True,     SCREAMING_SNAKE_CASE_						:					Optional[int]=True,     SCREAMING_SNAKE_CASE_						:					List[Any]=True,     SCREAMING_SNAKE_CASE_						:					str=99,     SCREAMING_SNAKE_CASE_						:					Optional[Any]=32,     SCREAMING_SNAKE_CASE_						:					Optional[int]=5,     SCREAMING_SNAKE_CASE_						:					str=4,     SCREAMING_SNAKE_CASE_						:					Dict=37,     SCREAMING_SNAKE_CASE_						:					Optional[Any]="gelu",     SCREAMING_SNAKE_CASE_						:					List[str]=0.1,     SCREAMING_SNAKE_CASE_						:					Tuple=0.1,     SCREAMING_SNAKE_CASE_						:					Optional[Any]=5_12,     SCREAMING_SNAKE_CASE_						:					Dict=16,     SCREAMING_SNAKE_CASE_						:					Any=2,     SCREAMING_SNAKE_CASE_						:					Tuple=0.02,     SCREAMING_SNAKE_CASE_						:					int=3,     SCREAMING_SNAKE_CASE_						:					Dict=None,     SCREAMING_SNAKE_CASE_						:					List[str]=20,     SCREAMING_SNAKE_CASE_						:					List[Any]=30,     SCREAMING_SNAKE_CASE_						:					Optional[int]=2,     SCREAMING_SNAKE_CASE_						:					Any=2,     SCREAMING_SNAKE_CASE_						:					Optional[Any]=10,     ):
										snake_case							:  Union[str, Any]			=     parent
										snake_case							:  List[str]			=     batch_size
										snake_case							:  List[str]			=     seq_length
										snake_case							:  Optional[int]			=     is_training
										snake_case							:  Optional[Any]			=     use_input_mask
										snake_case							:  List[Any]			=     use_token_type_ids
										snake_case							:  int			=     use_labels
										snake_case							:  List[Any]			=     vocab_size
										snake_case							:  Tuple			=     hidden_size
										snake_case							:  List[str]			=     num_hidden_layers
										snake_case							:  List[Any]			=     num_attention_heads
										snake_case							:  int			=     intermediate_size
										snake_case							:  List[str]			=     hidden_act
										snake_case							:  int			=     hidden_dropout_prob
										snake_case							:  Dict			=     attention_probs_dropout_prob
										snake_case							:  Tuple			=     max_position_embeddings
										snake_case							:  List[str]			=     type_vocab_size
										snake_case							:  str			=     type_sequence_label_size
										snake_case							:  Optional[Any]			=     initializer_range
										snake_case							:  List[str]			=     num_labels
										snake_case							:  Union[str, Any]			=     scope
										snake_case							:  int			=     max_xpath_tag_unit_embeddings
										snake_case							:  Optional[Any]			=     max_xpath_subs_unit_embeddings
										snake_case							:  str			=     tag_pad_id
										snake_case							:  Dict			=     subs_pad_id
										snake_case							:  List[Any]			=     max_depth




				def 						__snake_case				(				self						:					Union[str, Any]   ):
										snake_case							:  Tuple			=     ids_tensor([self.batch_size, self.seq_length],     self.vocab_size   )

										snake_case							:  Tuple			=     ids_tensor(
										    [self.batch_size, self.seq_length, self.max_depth],     self.max_xpath_tag_unit_embeddings   )

										snake_case							:  Tuple			=     ids_tensor(
										    [self.batch_size, self.seq_length, self.max_depth],     self.max_xpath_subs_unit_embeddings   )

										snake_case							:  Dict			=     None
										if self.use_input_mask:
																snake_case							:  List[str]			=     ids_tensor([self.batch_size, self.seq_length],     vocab_size=2   )

										snake_case							:  Any			=     None
										if self.use_token_type_ids:
																snake_case							:  Optional[int]			=     ids_tensor([self.batch_size, self.seq_length],     self.type_vocab_size   )

										snake_case							:  List[Any]			=     None
										snake_case							:  str			=     None
										if self.use_labels:
																snake_case							:  List[str]			=     ids_tensor([self.batch_size],     self.type_sequence_label_size   )
																snake_case							:  Union[str, Any]			=     ids_tensor([self.batch_size, self.seq_length],     self.num_labels   )

										snake_case							:  Union[str, Any]			=     self.get_config()

										return (
										    config,
										    input_ids,
										    xpath_tags_seq,
										    xpath_subs_seq,
										    token_type_ids,
										    input_mask,
										    sequence_labels,
										    token_labels,
										)




				def 						__snake_case				(				self						:					Tuple   ):
										return MarkupLMConfig(
										    vocab_size=self.vocab_size,     hidden_size=self.hidden_size,     num_hidden_layers=self.num_hidden_layers,     num_attention_heads=self.num_attention_heads,     intermediate_size=self.intermediate_size,     hidden_act=self.hidden_act,     hidden_dropout_prob=self.hidden_dropout_prob,     attention_probs_dropout_prob=self.attention_probs_dropout_prob,     max_position_embeddings=self.max_position_embeddings,     type_vocab_size=self.type_vocab_size,     initializer_range=self.initializer_range,     max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings,     max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings,     tag_pad_id=self.tag_pad_id,     subs_pad_id=self.subs_pad_id,     max_depth=self.max_depth,     )




				def 						__snake_case				(				self						:					Tuple,     SCREAMING_SNAKE_CASE_						:					Tuple,     SCREAMING_SNAKE_CASE_						:					int,     SCREAMING_SNAKE_CASE_						:					Any,     SCREAMING_SNAKE_CASE_						:					int,     SCREAMING_SNAKE_CASE_						:					Union[str, Any],     SCREAMING_SNAKE_CASE_						:					List[Any],     SCREAMING_SNAKE_CASE_						:					int,     SCREAMING_SNAKE_CASE_						:					int,     ):
										snake_case							:  Optional[Any]			=     MarkupLMModel(config=SCREAMING_SNAKE_CASE_   )
										model.to(SCREAMING_SNAKE_CASE_   )
										model.eval()
										print('''Configs:''',     model.config.tag_pad_id,     model.config.subs_pad_id   )
										snake_case							:  Optional[Any]			=     model(SCREAMING_SNAKE_CASE_,     attention_mask=SCREAMING_SNAKE_CASE_,     token_type_ids=SCREAMING_SNAKE_CASE_   )
										snake_case							:  Union[str, Any]			=     model(SCREAMING_SNAKE_CASE_,     token_type_ids=SCREAMING_SNAKE_CASE_   )
										snake_case							:  Dict			=     model(SCREAMING_SNAKE_CASE_   )
										self.parent.assertEqual(result.last_hidden_state.shape,     (self.batch_size, self.seq_length, self.hidden_size)   )
										self.parent.assertEqual(result.pooler_output.shape,     (self.batch_size, self.hidden_size)   )




				def 						__snake_case				(				self						:					Tuple,     SCREAMING_SNAKE_CASE_						:					int,     SCREAMING_SNAKE_CASE_						:					Any,     SCREAMING_SNAKE_CASE_						:					Tuple,     SCREAMING_SNAKE_CASE_						:					List[str],     SCREAMING_SNAKE_CASE_						:					str,     SCREAMING_SNAKE_CASE_						:					Any,     SCREAMING_SNAKE_CASE_						:					List[Any],     SCREAMING_SNAKE_CASE_						:					Tuple,     ):
										snake_case							:  int			=     self.num_labels
										snake_case							:  Union[str, Any]			=     MarkupLMForSequenceClassification(SCREAMING_SNAKE_CASE_   )
										model.to(SCREAMING_SNAKE_CASE_   )
										model.eval()
										snake_case							:  str			=     model(
										    SCREAMING_SNAKE_CASE_,     xpath_tags_seq=SCREAMING_SNAKE_CASE_,     xpath_subs_seq=SCREAMING_SNAKE_CASE_,     attention_mask=SCREAMING_SNAKE_CASE_,     token_type_ids=SCREAMING_SNAKE_CASE_,     labels=SCREAMING_SNAKE_CASE_,     )
										self.parent.assertEqual(result.logits.shape,     (self.batch_size, self.num_labels)   )




				def 						__snake_case				(				self						:					Optional[Any],     SCREAMING_SNAKE_CASE_						:					Optional[int],     SCREAMING_SNAKE_CASE_						:					str,     SCREAMING_SNAKE_CASE_						:					int,     SCREAMING_SNAKE_CASE_						:					Optional[Any],     SCREAMING_SNAKE_CASE_						:					str,     SCREAMING_SNAKE_CASE_						:					int,     SCREAMING_SNAKE_CASE_						:					List[str],     SCREAMING_SNAKE_CASE_						:					int,     ):
										snake_case							:  Any			=     self.num_labels
										snake_case							:  List[str]			=     MarkupLMForTokenClassification(config=SCREAMING_SNAKE_CASE_   )
										model.to(SCREAMING_SNAKE_CASE_   )
										model.eval()
										snake_case							:  Dict			=     model(
										    SCREAMING_SNAKE_CASE_,     xpath_tags_seq=SCREAMING_SNAKE_CASE_,     xpath_subs_seq=SCREAMING_SNAKE_CASE_,     attention_mask=SCREAMING_SNAKE_CASE_,     token_type_ids=SCREAMING_SNAKE_CASE_,     labels=SCREAMING_SNAKE_CASE_,     )
										self.parent.assertEqual(result.logits.shape,     (self.batch_size, self.seq_length, self.num_labels)   )




				def 						__snake_case				(				self						:					List[str],     SCREAMING_SNAKE_CASE_						:					Any,     SCREAMING_SNAKE_CASE_						:					int,     SCREAMING_SNAKE_CASE_						:					List[str],     SCREAMING_SNAKE_CASE_						:					Tuple,     SCREAMING_SNAKE_CASE_						:					Union[str, Any],     SCREAMING_SNAKE_CASE_						:					Any,     SCREAMING_SNAKE_CASE_						:					List[Any],     SCREAMING_SNAKE_CASE_						:					int,     ):
										snake_case							:  List[Any]			=     MarkupLMForQuestionAnswering(config=SCREAMING_SNAKE_CASE_   )
										model.to(SCREAMING_SNAKE_CASE_   )
										model.eval()
										snake_case							:  Union[str, Any]			=     model(
										    SCREAMING_SNAKE_CASE_,     xpath_tags_seq=SCREAMING_SNAKE_CASE_,     xpath_subs_seq=SCREAMING_SNAKE_CASE_,     attention_mask=SCREAMING_SNAKE_CASE_,     token_type_ids=SCREAMING_SNAKE_CASE_,     start_positions=SCREAMING_SNAKE_CASE_,     end_positions=SCREAMING_SNAKE_CASE_,     )
										self.parent.assertEqual(result.start_logits.shape,     (self.batch_size, self.seq_length)   )
										self.parent.assertEqual(result.end_logits.shape,     (self.batch_size, self.seq_length)   )




				def 						__snake_case				(				self						:					Optional[Any]   ):
										snake_case							:  int			=     self.prepare_config_and_inputs()
										(
										    (
										    snake_case
										),       (
										    snake_case
										),       (
										    snake_case
										),       (
										    snake_case
										),       (
										    snake_case
										),       (
										    snake_case
										),       (
										    snake_case
										),       (
										    snake_case
										),       
										)							:  Any			=     config_and_inputs
										snake_case							:  str			=     {
										    '''input_ids''': input_ids,
										    '''xpath_tags_seq''': xpath_tags_seq,
										    '''xpath_subs_seq''': xpath_subs_seq,
										    '''token_type_ids''': token_type_ids,
										    '''attention_mask''': input_mask,
										}
										return config, inputs_dict




@require_torch
class     a (					__magic_name__					,__magic_name__					,unittest.TestCase     ):
				_snake_case		     =			(
				    (
				        MarkupLMModel,
				        MarkupLMForSequenceClassification,
				        MarkupLMForTokenClassification,
				        MarkupLMForQuestionAnswering,
				    )
				    if is_torch_available()
				    else None
				)
				_snake_case		     =			(
				    {
				        '''feature-extraction''': MarkupLMModel,
				        '''question-answering''': MarkupLMForQuestionAnswering,
				        '''text-classification''': MarkupLMForSequenceClassification,
				        '''token-classification''': MarkupLMForTokenClassification,
				        '''zero-shot''': MarkupLMForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)




				def 						__snake_case				(				self						:					Optional[int],     SCREAMING_SNAKE_CASE_						:					List[str],     SCREAMING_SNAKE_CASE_						:					Optional[int],     SCREAMING_SNAKE_CASE_						:					List[str],     SCREAMING_SNAKE_CASE_						:					Dict,     SCREAMING_SNAKE_CASE_						:					List[Any]   ):
										# ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]`
										# (batch of pretokenized examples).
										return True




				def 						__snake_case				(				self						:					str   ):
										snake_case							:  Tuple			=     MarkupLMModelTester(self   )
										snake_case							:  List[Any]			=     ConfigTester(self,     config_class=SCREAMING_SNAKE_CASE_,     hidden_size=37   )




				def 						__snake_case				(				self						:					Any   ):
										self.config_tester.run_common_tests()




				def 						__snake_case				(				self						:					Optional[int]   ):
										snake_case							:  Dict			=     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_   )




				def 						__snake_case				(				self						:					List[Any]   ):
										snake_case							:  List[Any]			=     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE_   )




				def 						__snake_case				(				self						:					List[Any]   ):
										snake_case							:  Dict			=     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE_   )




				def 						__snake_case				(				self						:					Union[str, Any]   ):
										snake_case							:  Dict			=     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE_   )




def   A			(							):
						snake_case							:  Optional[int]			=     '''
    <!DOCTYPE html>
    <html>
    <head>
    <title>Page Title</title>
    </head>
    <body>

    <h1>This is a Heading</h1>
    <p>This is a paragraph.</p>

    </body>
    </html>
    '''

						return html_string





@require_torch
class     a (					unittest.TestCase     ):




				@cached_property
				def 						__snake_case				(				self						:					Tuple   ):
										# TODO use from_pretrained here
										snake_case							:  Optional[int]			=     MarkupLMFeatureExtractor()
										snake_case							:  Tuple			=     MarkupLMTokenizer.from_pretrained('''microsoft/markuplm-base'''   )

										return MarkupLMProcessor(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_   )




				@slow
				def 						__snake_case				(				self						:					Dict   ):
										snake_case							:  Dict			=     MarkupLMModel.from_pretrained('''microsoft/markuplm-base'''   ).to(SCREAMING_SNAKE_CASE_   )

										snake_case							:  List[str]			=     self.default_processor

										snake_case							:  str			=     processor(prepare_html_string(),     return_tensors='''pt'''   )
										snake_case							:  List[Any]			=     inputs.to(SCREAMING_SNAKE_CASE_   )

										# forward pass
										with torch.no_grad():
																snake_case							:  Any			=     model(**SCREAMING_SNAKE_CASE_   )

										# verify the last hidden states
										snake_case							:  Tuple			=     torch.Size([1, 14, 7_68]   )
										self.assertEqual(outputs.last_hidden_state.shape,     SCREAMING_SNAKE_CASE_   )

										snake_case							:  Union[str, Any]			=     torch.tensor(
										    [[0.0675, -0.0052, 0.5001], [-0.2281, 0.0802, 0.2192], [-0.0583, -0.3311, 0.1185]]   ).to(SCREAMING_SNAKE_CASE_   )

										self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3],     SCREAMING_SNAKE_CASE_,     atol=1e-4   )   )

