

import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
       from tests.trainer.test_trainer import (  # noqa
           RegressionModelConfig,
           RegressionPreTrainedModel,
           get_regression_trainer,
       )


set_seed(42)

lowerCamelCase     =      get_tests_dir('''fixtures''')
lowerCamelCase     =      os.path.join(dirname(get_tests_dir()))
lowerCamelCase     =      dirname(os.path.abspath(__file__))

# default torch.distributed port
lowerCamelCase     =      """10999"""

lowerCamelCase     =      """t5-small"""

# *** Working Models ***
lowerCamelCase     =      """hf-internal-testing/tiny-albert"""
lowerCamelCase     =      """sshleifer/bart-tiny-random"""
lowerCamelCase     =      """hf-internal-testing/tiny-bert"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-bigbird_pegasus"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-big_bird"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-blenderbot"""
lowerCamelCase     =      """bigscience/bigscience-small-testing"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-deberta"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-deberta-v2"""
lowerCamelCase     =      """sshleifer/tiny-distilbert-base-cased"""
lowerCamelCase     =      """hf-internal-testing/tiny-electra"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-flaubert"""
lowerCamelCase     =      """stas/tiny-wmt19-en-de"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-funnel"""
lowerCamelCase     =      """sshleifer/tiny-gpt2"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-gptj"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-gpt_neo"""
lowerCamelCase     =      """hf-internal-testing/tiny-layoutlm"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-led"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-longformer"""
lowerCamelCase     =      """stas/tiny-m2m_100"""  # hf tiny model is unsuitable
lowerCamelCase     =      """sshleifer/tiny-marian-en-de"""
lowerCamelCase     =      """sshleifer/tiny-mbart"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-mobilebert"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-mpnet"""
lowerCamelCase     =      """stas/pegasus-cnn_dailymail-tiny-random"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-prophetnet"""
lowerCamelCase     =      """sshleifer/tiny-distilroberta-base"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-squeezebert"""
lowerCamelCase     =      """patrickvonplaten/t5-tiny-random"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-t5-v1.1"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-vit"""
lowerCamelCase     =      """hf-internal-testing/tiny-xlm-roberta"""
lowerCamelCase     =      """sshleifer/tiny-xlnet-base-cased"""


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
lowerCamelCase     =      """hf-internal-testing/tiny-random-mt5"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-camembert"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-openai-gpt"""

# missing tokenizer files
lowerCamelCase     =      """hf-internal-testing/tiny-random-convbert"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-layoutlmv2"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-hubert"""

# issues with tokenizer
lowerCamelCase     =      """hf-internal-testing/tiny-random-ctrl"""
lowerCamelCase     =      """hf-internal-testing/tiny-random-transfo-xl"""  # same as ctrl

# other issues with tiny models
lowerCamelCase     =      """hf-internal-testing/tiny-random-ibert"""  # multiple issues with either mlm/qa/clas
lowerCamelCase     =      """hf-internal-testing/tiny-random-reformer"""  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
lowerCamelCase     =      """hf-internal-testing/tiny-random-dpr"""
# - "dpr"  examples/research_projects/rag-end2end-retriever/
lowerCamelCase     =      """hf-internal-testing/tiny-random-rag"""
# - "rag" research_projects
lowerCamelCase     =      """"""
# - "luke" Entities classes - no plan to make such example
lowerCamelCase     =      """hf-internal-testing/tiny-random-lxmert"""
# - "lxmert" doesn't work with run_qa.py
lowerCamelCase     =      """hf-internal-testing/tiny-random-clip"""
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
lowerCamelCase     =      """hf-internal-testing/tiny-random-speech_to_text"""
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
lowerCamelCase     =      """hf-internal-testing/tiny-random-tapas"""




def     UpperCAmelCase__				(    _A   :				str=False							):





  '''simple docstring'''







  a__						=min(2			,   get_gpu_count()							) if distributed else 1
  a__						=os.environ.get('''DS_TEST_PORT'''			,   __lowerCamelCase							)
  return F"""deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}""".split()




def     UpperCAmelCase__				(    ):





  '''simple docstring'''







  a__						=F"""{FIXTURE_DIRECTORY}/tests_samples"""
  a__						=F"""{data_dir_samples}/wmt_en_ro"""
  a__						=F"""{data_dir_samples}/xsum"""
  a__						="""
        --do_train
        --max_train_samples 4
        --per_device_train_batch_size 2
        --num_train_epochs 1
        --fp16
        --report_to none
        --overwrite_output_dir
        """.split()

  # try to cover as many models as possible once (it's enough to run on one task per model)
  # but need a tiny model for each
  #
  # should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
  a__						={
      """trans""": [
          """bart""",
          """fsmt""",
          """m2m_100""",
          """marian""",
          """mbart""",
          """t5""",
          """t5_v1""",
          # "mt5", missing model files
      ],
      """sum""": [
          """pegasus""",
      ],
      """clm""": [
          """big_bird""",
          """bigbird_pegasus""",
          """blenderbot""",
          """bloom""",
          """gpt2""",
          """gpt_neo""",
          """gptj""",
          """xlm-roberta""",
          """prophetnet""",
          # "camembert", missing model files
      ],
      """mlm""": [
          """albert""",
          """deberta""",
          """deberta-v2""",
          """distilbert""",
          """electra""",
          """flaubert""",
          """funnel""",
          """layoutlm""",
          # "reformer", # multiple issues with either mlm/qa/clas
      ],
      """qa""": [
          """led""",
          """longformer""",
          """mobilebert""",
          """mpnet""",
          """roberta""",
          """squeezebert""",
          # "convbert", # missing tokenizer files
          # "layoutlmv2", missing model files
      ],
      """clas""": [
          """bert""",
          """xlnet""",
          # "hubert", # missing tokenizer files
          # "ibert", # multiple issues with either mlm/qa/clas
          # "transfo-xl", # tokenizer issues as ctrl
          # "ctrl", # tokenizer issues
          # "openai-gpt", missing model files
          # "tapas", multiple issues
      ],
      """img_clas""": [
          """vit""",
      ],
  }

  a__						=F"""{ROOT_DIRECTORY}/examples/pytorch"""

  a__						={
      """trans""": F"""\n        {scripts_dir}/translation/run_translation.py\n        --train_file {data_dir_wmt}/train.json\n        --source_lang en\n        --target_lang ro\n        """,
      """sum""": F"""\n        {scripts_dir}/summarization/run_summarization.py\n        --train_file {data_dir_xsum}/sample.json\n        --max_source_length 12\n        --max_target_length 12\n        --lang en\n        """,
      """clm""": F"""\n        {scripts_dir}/language-modeling/run_clm.py\n        --train_file {FIXTURE_DIRECTORY}/sample_text.txt\n        --block_size 8\n        """,
      """mlm""": F"""\n        {scripts_dir}/language-modeling/run_mlm.py\n        --train_file {FIXTURE_DIRECTORY}/sample_text.txt\n        """,
      """qa""": F"""\n        {scripts_dir}/question-answering/run_qa.py\n        --train_file {data_dir_samples}/SQUAD/sample.json\n        """,
      """clas""": F"""\n        {scripts_dir}/text-classification/run_glue.py\n        --train_file {data_dir_samples}/MRPC/train.csv\n        --max_seq_length 12\n        --task_name MRPC\n        """,
      """img_clas""": F"""\n        {scripts_dir}/image-classification/run_image_classification.py\n            --dataset_name hf-internal-testing/cats_vs_dogs_sample\n            --remove_unused_columns False\n            --max_steps 10\n            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json\n        """,
  }

  a__						=get_launcher(distributed=__lowerCamelCase							)

  a__						={}
  for task, args in tasks.items():
    a__						=args.split()
    for model in tasksamodels[task]:
      a__						=globals()[F"""{model.upper().replace("-"			,   "_"							)}_TINY"""]
      a__						=F"""--model_name_or_path {model_name}""".split()
      a__						=launcher + args + args_model + args_main

      # # generation special case
      # if task == "gen":
      #     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
      #     args_model += f"--model_type {model}".split()
      #     cmds[f"{task}_{model}"] = launcher + args + args_model
      # else:

  return cmds


lowerCamelCase     =      make_task_cmds()

lowerCamelCase     =      """zero2"""
lowerCamelCase     =      """zero3"""

lowerCamelCase     =      [ZEROa, ZEROa]




def     UpperCAmelCase__				(    _A   :				Optional[Any]			,   _A   :				List[Any]			,   _A   :				List[str]							):





  '''simple docstring'''







  a__						=parameterized.to_safe_name('''_'''.join(str(__lowerCamelCase							) for x in param.args							)							)
  return F"""{func.__name__}_{param_based_name}"""


# Cartesian-product of zero stages with models to test
lowerCamelCase     =      list(itertools.product(stages, task_cmds.keys()))







@slow
@require_deepspeed
@require_torch_gpu
class 		__magic_name__    (							__lowerCamelCase							):

   '''simple docstring'''




   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_ )							->					Optional[Any]:






     """simple docstring"""






     if task not in task_cmds:
       raise ValueError(F"""don't know of task {task}, have {task_cmds.keys()}""" )

     a__						=task_cmds[task]
     a__						=F"""--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json""".split()

     a__						=self.get_auto_remove_tmp_dir()
     a__						=F"""--output_dir {output_dir}""".split()

     cmd += args_ds + args_out

     return cmd, output_dir



   @parameterized.expand(_lowerCAmelCase,					name_func=_lowerCAmelCase )
   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_ )							->					Union[str, Any]:






     """simple docstring"""






     a__						=self.get_task_cmd(_lowerCAmelCase,					_lowerCAmelCase )

     # 1. generate the checkpoint
     cmd += "--save_steps 1".split()
     # keep for quick debug
     # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
     execute_subprocess_async(_lowerCAmelCase,					env=self.get_env() )

     # 2. test that the fp32 weights get reconsolidated
     a__						=F"""{output_dir}/checkpoint-1"""
     a__						=F"""{chkpt_dir}/out.bin"""
     a__						=F"""{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}"""
     # keep for quick debug
     # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
     subprocess.check_call(_lowerCAmelCase,					shell=_lowerCAmelCase )
     assert os.path.exists(_lowerCAmelCase ), F"""{recovered_model_path} was not found"""

     # possibly could also test that the resulting saved model is usable but given that we use
     # random models we won't know if it's any good



import unittest

from transformers import ErnieMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           ErnieMForInformationExtraction,
           ErnieMForMultipleChoice,
           ErnieMForQuestionAnswering,
           ErnieMForSequenceClassification,
           ErnieMForTokenClassification,
           ErnieMModel,
       )
       from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST







class 		__magic_name__    :

   '''simple docstring'''




   def __init__(       self,					lowercase_,					lowercase_=13,					lowercase_=7,					lowercase_=True,					lowercase_=True,					lowercase_=True,					lowercase_=99,					lowercase_=32,					lowercase_=5,					lowercase_=4,					lowercase_=37,					lowercase_="gelu",					lowercase_=0.1,					lowercase_=0.1,					lowercase_=512,					lowercase_=16,					lowercase_=2,					lowercase_=0.02,					lowercase_=3,					lowercase_=4,					lowercase_=None,					)							->					List[Any]:






     """simple docstring"""






     a__						=parent
     a__						=batch_size
     a__						=seq_length
     a__						=is_training
     a__						=use_input_mask
     a__						=use_labels
     a__						=vocab_size
     a__						=hidden_size
     a__						=num_hidden_layers
     a__						=num_attention_heads
     a__						=intermediate_size
     a__						=hidden_act
     a__						=hidden_dropout_prob
     a__						=attention_probs_dropout_prob
     a__						=max_position_embeddings
     a__						=type_vocab_size
     a__						=type_sequence_label_size
     a__						=initializer_range
     a__						=num_labels
     a__						=num_choices
     a__						=scope



   def 				_UpperCAmelCase				(       self )							->					List[Any]:






     """simple docstring"""






     a__						=ids_tensor([self.batch_size, self.seq_length],					self.vocab_size )

     a__						=None
     if self.use_input_mask:
       a__						=random_attention_mask([self.batch_size, self.seq_length] )

     a__						=None
     a__						=None
     a__						=None
     if self.use_labels:
       a__						=ids_tensor([self.batch_size],					self.type_sequence_label_size )
       a__						=ids_tensor([self.batch_size, self.seq_length],					self.num_labels )
       a__						=ids_tensor([self.batch_size],					self.num_choices )

     a__						=self.get_config()

     return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels



   def 				_UpperCAmelCase				(       self )							->					Optional[Any]:






     """simple docstring"""






     a__						=ids_tensor([self.batch_size, self.seq_length],					self.vocab_size )

     a__						=None
     if self.use_input_mask:
       a__						=random_attention_mask([self.batch_size, self.seq_length] )
     a__						=self.get_config()

     return config, input_ids, input_mask



   def 				_UpperCAmelCase				(       self )							->					Optional[int]:






     """simple docstring"""






     return ErnieMConfig(
         vocab_size=self.vocab_size,					hidden_size=self.hidden_size,					num_hidden_layers=self.num_hidden_layers,					num_attention_heads=self.num_attention_heads,					intermediate_size=self.intermediate_size,					hidden_act=self.hidden_act,					hidden_dropout_prob=self.hidden_dropout_prob,					attention_probs_dropout_prob=self.attention_probs_dropout_prob,					max_position_embeddings=self.max_position_embeddings,					type_vocab_size=self.type_vocab_size,					initializer_range=self.initializer_range,					)



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					Union[str, Any]:






     """simple docstring"""






     a__						=ErnieMModel(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=model(lowercase_,					return_dict=lowercase_ )
     self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.hidden_size) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					List[str]:






     """simple docstring"""






     a__						=ErnieMForQuestionAnswering(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=model(
         lowercase_,					attention_mask=lowercase_,					start_positions=lowercase_,					end_positions=lowercase_,					)
     self.parent.assertEqual(result.start_logits.shape,					(self.batch_size, self.seq_length) )
     self.parent.assertEqual(result.end_logits.shape,					(self.batch_size, self.seq_length) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					Dict:






     """simple docstring"""






     a__						=ErnieMForInformationExtraction(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=torch.ones_like(lowercase_,					dtype=torch.floataa )
     a__						=model(
         lowercase_,					attention_mask=lowercase_,					start_positions=lowercase_,					end_positions=lowercase_,					)
     self.parent.assertEqual(result.start_logits.shape,					(self.batch_size, self.seq_length) )
     self.parent.assertEqual(result.end_logits.shape,					(self.batch_size, self.seq_length) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					Union[str, Any]:






     """simple docstring"""






     a__						=self.num_labels
     a__						=ErnieMForSequenceClassification(lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=model(lowercase_,					attention_mask=lowercase_,					labels=lowercase_ )
     self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.num_labels) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					Any:






     """simple docstring"""






     a__						=self.num_labels
     a__						=ErnieMForTokenClassification(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     input_ids.to(lowercase_ )
     input_mask.to(lowercase_ )
     token_labels.to(lowercase_ )

     a__						=model(lowercase_,					attention_mask=lowercase_,					labels=lowercase_ )

     self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.num_labels) )



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					Optional[int]:






     """simple docstring"""






     a__						=self.num_choices
     a__						=ErnieMForMultipleChoice(config=lowercase_ )
     model.to(lowercase_ )
     model.eval()
     a__						=input_ids.unsqueeze(1 ).expand(-1,					self.num_choices,					-1 ).contiguous()
     a__						=input_mask.unsqueeze(1 ).expand(-1,					self.num_choices,					-1 ).contiguous()
     a__						=model(
         lowercase_,					attention_mask=lowercase_,					labels=lowercase_,					)
     self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.num_choices) )



   def 				_UpperCAmelCase				(       self )							->					Optional[int]:






     """simple docstring"""






     a__						=self.prepare_config_and_inputs()
     (
         (
         a__
     ),   (
         a__
     ),   (
         a__
     ),   (
         a__
     ),   (
         a__
     ),   (
         a__
     ),   
     )						=config_and_inputs
     a__						={'''input_ids''': input_ids, '''attention_mask''': input_mask}
     return config, inputs_dict







@require_torch
class 		__magic_name__    (							lowerCamelCase__	, lowerCamelCase__	, unittest.TestCase							):

   '''simple docstring'''




   lowerCamelCase__				:			List[Any]												=     (
       (
           ErnieMModel,
           ErnieMForMultipleChoice,
           ErnieMForQuestionAnswering,
           ErnieMForSequenceClassification,
           ErnieMForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   lowerCamelCase__				:			Union[str, Any]												=     ()
   lowerCamelCase__				:			Union[str, Any]												=     (
       {
           'feature-extraction': ErnieMModel,
           'question-answering': ErnieMForQuestionAnswering,
           'text-classification': ErnieMForSequenceClassification,
           'token-classification': ErnieMForTokenClassification,
           'zero-shot': ErnieMForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   lowerCamelCase__				:			Any												=     False



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					Optional[Any]:






     """simple docstring"""






     if pipeline_test_casse_name == "QAPipelineTests":
       return True

     return False



   def 				_UpperCAmelCase				(       self )							->					int:






     """simple docstring"""






     a__						=ErnieMModelTester(self )
     a__						=ConfigTester(self,					config_class=lowercase_,					hidden_size=37 )



   def 				_UpperCAmelCase				(       self )							->					Any:






     """simple docstring"""






     self.config_tester.run_common_tests()



   def 				_UpperCAmelCase				(       self )							->					Any:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					Union[str, Any]:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     for type in ["absolute", "relative_key", "relative_key_query"]:
       a__						=type
       self.model_tester.create_and_check_model(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					Dict:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					Tuple:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					Any:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					str:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_information_extraction(*lowercase_ )



   def 				_UpperCAmelCase				(       self )							->					Any:






     """simple docstring"""






     a__						=self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*lowercase_ )



   @slow
   def 				_UpperCAmelCase				(       self )							->					str:






     """simple docstring"""






     for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       a__						=ErnieMModel.from_pretrained(lowercase_ )
       self.assertIsNotNone(lowercase_ )







@require_torch
class 		__magic_name__    (							unittest.TestCase							):

   '''simple docstring'''




   @slow
   def 				_UpperCAmelCase				(       self )							->					str:






     """simple docstring"""






     a__						=ErnieMModel.from_pretrained('''susnato/ernie-m-base_pytorch''' )
     model.eval()
     a__						=torch.tensor([[0, 1, 2, 3, 4, 5]] )
     a__						=model(lowercase_ )[0]

     # TODO Replace vocab size
     a__						=768

     a__						=torch.Size((1, 6, hidden_size) )
     self.assertEqual(output.shape,					lowercase_ )

     a__						=torch.tensor(
         [[[-0.0012, 0.1245, -0.0214], [-0.0742, 0.0244, -0.0771], [-0.0333, 0.1164, -0.1554]]] )

     self.assertTrue(torch.allclose(output[:, :3, :3],					lowercase_,					atol=1E-3 ) )
