import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


lowerCAmelCase      :				Union[str, Any]     			={0: "null", 1: "short", 2: "long", 3: "yes", 4: "no"}
lowerCAmelCase      :				Optional[int]     			=set("".join(["‘", "’", "´", "`", ".", ",", "-", "\""]))







def     A__	(       __A ,			__A=0 ,			__A=None						):







    '''simple docstring'''
    return [" ".join(x.split(""" """						)[begin:end]						) for x in answers if len(x.split(""" """						)						) > 1]







def     A__	(       __A ,			__A=False						):







    '''simple docstring'''
    if make_sub_answers:
        # if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
        # *e.g.* if the correct answer contains a prefix such as "the", or "a"
        _lowerCamelCase  :		Dict			    =    (
            given_answers + get_sub_answers(__A ,			begin=1						) + get_sub_answers(__A ,			end=-1						)
        )
    _lowerCamelCase  :		Tuple			    =    []
    for answer in given_answers:
        _lowerCamelCase  :		int			    =    answer.replace("""_""" ,			""" """						).lower()
        _lowerCamelCase  :		Union[str, Any]			    =    """""".join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else """ """ for c in alias						)
        answers.append(""" """.join(alias.split()						).strip()						)
    return set(__A						)







def     A__	(       __A ,			__A ,			__A=1 ,			__A=100						):







    '''simple docstring'''
    _lowerCamelCase    ,				_lowerCamelCase  :		List[Any]			    =    jax.lax.top_k(__A ,			__A						)
    _lowerCamelCase    ,				_lowerCamelCase  :		int			    =    jax.lax.top_k(__A ,			__A						)

    _lowerCamelCase  :		List[Any]			    =    best_end_idx[:, None] - best_start_idx[None, :]
    _lowerCamelCase  :		Optional[Any]			    =    jnp.logical_or(widths < 0 ,			widths > max_size						)
    _lowerCamelCase  :		Union[str, Any]			    =    (best_end_scores[:, None] + best_start_scores[None, :]) - (1E8 * mask)
    _lowerCamelCase  :		Tuple			    =    jnp.argmax(__A						).item()

    return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]







def     A__	(       __A						):







    '''simple docstring'''
    _lowerCamelCase  :		Optional[Any]			    =    sample["""question"""]["""text"""]
    _lowerCamelCase  :		Dict			    =    sample["""document"""]["""tokens"""]["""token"""]
    _lowerCamelCase  :		Optional[int]			    =    sample["""document"""]["""tokens"""]["""is_html"""]
    _lowerCamelCase  :		Optional[int]			    =    sample["""annotations"""]["""long_answer"""]
    _lowerCamelCase  :		Optional[int]			    =    sample["""annotations"""]["""short_answers"""]

    _lowerCamelCase  :		List[str]			    =    """ """.join([context[i] for i in range(len(__A						)						) if not is_html[i]]						)

    # 0 - No ; 1 - Yes
    for answer in sample["annotations"]["yes_no_answer"]:
        if answer == 0 or answer == 1:
            return {
                "question": question,
                "context": context_string,
                "short": [],
                "long": [],
                "category": "no" if answer == 0 else "yes",
            }

    _lowerCamelCase  :		str			    =    []
    for s in short_answers:
        short_targets.extend(s["""text"""]						)
    _lowerCamelCase  :		List[str]			    =    list(set(__A						)						)

    _lowerCamelCase  :		Optional[int]			    =    []
    for s in long_answers:
        if s["start_token"] == -1:
            continue
        _lowerCamelCase  :		Tuple			    =    context[s["""start_token"""] : s["""end_token"""]]
        _lowerCamelCase  :		List[Any]			    =    is_html[s["""start_token"""] : s["""end_token"""]]
        _lowerCamelCase  :		List[str]			    =    """ """.join([answer[i] for i in range(len(__A						)						) if not html[i]]						)
        if new_answer not in long_targets:
            long_targets.append(__A						)

    _lowerCamelCase  :		int			    =    """long_short""" if len(short_targets + long_targets						) > 0 else """null"""

    return {
        "question": question,
        "context": context_string,
        "short": short_targets,
        "long": long_targets,
        "category": category,
    }







def     A__	(       ):







    '''simple docstring'''
    _lowerCamelCase  :		Optional[int]			    =    load_from_disk("""natural-questions-validation"""						)
    _lowerCamelCase  :		Optional[int]			    =    dataset.map(__A						).remove_columns(["""annotations""", """document""", """id"""]						)
    print(__A						)

    _lowerCamelCase  :		Optional[Any]			    =    dataset.filter(lambda __A						: (len(x["""question"""]						) + len(x["""context"""]						)) < 4 * 4_096						)
    _lowerCamelCase  :		Union[str, Any]			    =    short_validation_dataset.filter(lambda __A						: x["category"] != "null"						)
    short_validation_dataset

    _lowerCamelCase  :		Any			    =    """vasudevgupta/flax-bigbird-natural-questions"""
    _lowerCamelCase  :		Any			    =    FlaxBigBirdForNaturalQuestions.from_pretrained(__A						)
    _lowerCamelCase  :		Union[str, Any]			    =    BigBirdTokenizerFast.from_pretrained(__A						)

    @jax.jit
    def forward(*__A ,			**__A						):
        _lowerCamelCase    ,				_lowerCamelCase    ,				_lowerCamelCase  :		Union[str, Any]			    =    model(*__A ,			**__A						)
        return start_logits, end_logits, jnp.argmax(__A ,			axis=-1						)

    def evaluate(__A						):
        # encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
        _lowerCamelCase  :		Dict			    =    tokenizer(
            example["""question"""] ,			example["""context"""] ,			return_tensors="""np""" ,			max_length=4_096 ,			padding="""max_length""" ,			truncation=__A ,			)

        _lowerCamelCase    ,				_lowerCamelCase    ,				_lowerCamelCase  :		Optional[int]			    =    forward(**__A						)

        _lowerCamelCase  :		Tuple			    =    CATEGORY_MAPPING[category.item()]

        _lowerCamelCase  :		Any			    =    example["""long"""] + example["""short"""]
        if example["category"] in ["yes", "no", "null"]:
            _lowerCamelCase  :		int			    =    [example["""category"""]]
        _lowerCamelCase  :		Optional[int]			    =    example["""category"""] != """null"""
        # Now target can be: "yes", "no", "null", "list of long & short answers"

        if predicted_category in ["yes", "no", "null"]:
            _lowerCamelCase  :		Union[str, Any]			    =    [predicted_category]
            _lowerCamelCase  :		List[str]			    =    example["""output"""] == example["""targets"""]
            _lowerCamelCase  :		str			    =    predicted_category != """null"""
            return example

        _lowerCamelCase  :		Dict			    =    38 if predicted_category == """short""" else 1_024
        _lowerCamelCase    ,				_lowerCamelCase  :		List[Any]			    =    get_best_valid_start_end_idx(
            start_scores[0] ,			end_scores[0] ,			top_k=8 ,			max_size=__A						)

        _lowerCamelCase  :		List[Any]			    =    inputs["""input_ids"""][0].tolist()
        _lowerCamelCase  :		List[Any]			    =    [tokenizer.decode(input_ids[start_score : end_score + 1]						)]

        _lowerCamelCase  :		List[str]			    =    expand_to_aliases(example["""targets"""] ,			make_sub_answers=__A						)
        _lowerCamelCase  :		Optional[Any]			    =    expand_to_aliases(example["""output"""]						)

        # some preprocessing to both prediction and answer
        _lowerCamelCase  :		int			    =    {"""""".join(a.split()						) for a in answers}
        _lowerCamelCase  :		Optional[Any]			    =    {"""""".join(p.split()						) for p in predictions}
        _lowerCamelCase  :		Dict			    =    {s for s in predictions if s not in ["""``""", """''""", """`""", """'"""]}

        # if there is a common element, it's a exact match
        _lowerCamelCase  :		Union[str, Any]			    =    len(list(answers & predictions						)						) > 0
        _lowerCamelCase  :		List[str]			    =    predicted_category != """null""" and len(__A						) > 0

        return example

    _lowerCamelCase  :		int			    =    short_validation_dataset.map(__A						)

    _lowerCamelCase  :		List[Any]			    =    len(__A						)
    _lowerCamelCase  :		List[str]			    =    len(short_validation_dataset.filter(lambda __A						: x["match"] == 1						)						)
    print("""EM score:""" ,			(matched / total) * 100 ,			"""%"""						)


if __name__ == "__main__":
    main()





import unittest

from transformers import YosoConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        YosoForMaskedLM,
        YosoForMultipleChoice,
        YosoForQuestionAnswering,
        YosoForSequenceClassification,
        YosoForTokenClassification,
        YosoModel,
    )
    from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST




class 						__snake_case						:





      '''simple docstring'''






      def __init__(       self  :					List[Any]     ,	_UpperCamelCase  :					Union[str, Any]     ,	_UpperCamelCase  :					Tuple=13     ,	_UpperCamelCase  :					Union[str, Any]=7     ,	_UpperCamelCase  :					Union[str, Any]=True     ,	_UpperCamelCase  :					Dict=True     ,	_UpperCamelCase  :					List[Any]=True     ,	_UpperCamelCase  :					Optional[Any]=True     ,	_UpperCamelCase  :					Optional[int]=99     ,	_UpperCamelCase  :					List[Any]=32     ,	_UpperCamelCase  :					Dict=5     ,	_UpperCamelCase  :					Dict=4     ,	_UpperCamelCase  :					Optional[int]=37     ,	_UpperCamelCase  :					List[Any]="gelu"     ,	_UpperCamelCase  :					int=0.1     ,	_UpperCamelCase  :					str=0.1     ,	_UpperCamelCase  :					int=512     ,	_UpperCamelCase  :					Optional[Any]=16     ,	_UpperCamelCase  :					Tuple=2     ,	_UpperCamelCase  :					Any=0.0_2     ,	_UpperCamelCase  :					Any=3     ,	_UpperCamelCase  :					Union[str, Any]=4     ,	_UpperCamelCase  :					List[Any]=None     ,	)					->List[Any]:


          """simple docstring"""


          _lowerCamelCase  :		List[str]			    =    parent
          _lowerCamelCase  :		List[Any]			    =    batch_size
          _lowerCamelCase  :		int			    =    seq_length
          _lowerCamelCase  :		List[Any]			    =    is_training
          _lowerCamelCase  :		List[Any]			    =    use_input_mask
          _lowerCamelCase  :		Tuple			    =    use_token_type_ids
          _lowerCamelCase  :		int			    =    use_labels
          _lowerCamelCase  :		int			    =    vocab_size
          _lowerCamelCase  :		List[str]			    =    hidden_size
          _lowerCamelCase  :		List[Any]			    =    num_hidden_layers
          _lowerCamelCase  :		str			    =    num_attention_heads
          _lowerCamelCase  :		List[str]			    =    intermediate_size
          _lowerCamelCase  :		List[Any]			    =    hidden_act
          _lowerCamelCase  :		Optional[int]			    =    hidden_dropout_prob
          _lowerCamelCase  :		Optional[int]			    =    attention_probs_dropout_prob
          _lowerCamelCase  :		Tuple			    =    max_position_embeddings
          _lowerCamelCase  :		List[str]			    =    type_vocab_size
          _lowerCamelCase  :		Dict			    =    type_sequence_label_size
          _lowerCamelCase  :		str			    =    initializer_range
          _lowerCamelCase  :		Tuple			    =    num_labels
          _lowerCamelCase  :		Union[str, Any]			    =    num_choices
          _lowerCamelCase  :		str			    =    scope



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Union[str, Any])					->str:


          """simple docstring"""


          _lowerCamelCase  :		Optional[int]			    =    ids_tensor([self.batch_size, self.seq_length]     ,	self.vocab_size)

          _lowerCamelCase  :		Tuple			    =    None
          if self.use_input_mask:
              _lowerCamelCase  :		Dict			    =    random_attention_mask([self.batch_size, self.seq_length])

          _lowerCamelCase  :		str			    =    None
          if self.use_token_type_ids:
              _lowerCamelCase  :		Optional[Any]			    =    ids_tensor([self.batch_size, self.seq_length]     ,	self.type_vocab_size)

          _lowerCamelCase  :		List[str]			    =    None
          _lowerCamelCase  :		Optional[int]			    =    None
          _lowerCamelCase  :		Dict			    =    None
          if self.use_labels:
              _lowerCamelCase  :		List[str]			    =    ids_tensor([self.batch_size]     ,	self.type_sequence_label_size)
              _lowerCamelCase  :		List[str]			    =    ids_tensor([self.batch_size, self.seq_length]     ,	self.num_labels)
              _lowerCamelCase  :		Any			    =    ids_tensor([self.batch_size]     ,	self.num_choices)

          _lowerCamelCase  :		Optional[Any]			    =    self.get_config()

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Optional[int])					->str:


          """simple docstring"""


          return YosoConfig(
              vocab_size=self.vocab_size     ,	hidden_size=self.hidden_size     ,	num_hidden_layers=self.num_hidden_layers     ,	num_attention_heads=self.num_attention_heads     ,	intermediate_size=self.intermediate_size     ,	hidden_act=self.hidden_act     ,	hidden_dropout_prob=self.hidden_dropout_prob     ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,	max_position_embeddings=self.max_position_embeddings     ,	type_vocab_size=self.type_vocab_size     ,	is_decoder=_UpperCamelCase     ,	initializer_range=self.initializer_range     ,	)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Union[str, Any])					->Optional[Any]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.get_config()
          _lowerCamelCase  :		List[str]			    =    300
          return config



      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[Any])					->Optional[Any]:


          """simple docstring"""


          (
              (
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				
          )  :		Dict			    =    self.prepare_config_and_inputs()

          _lowerCamelCase  :		int			    =    True
          _lowerCamelCase  :		List[Any]			    =    floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
          _lowerCamelCase  :		Any			    =    ids_tensor([self.batch_size, self.seq_length]     ,	vocab_size=2)

          return (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              encoder_hidden_states,
              encoder_attention_mask,
          )



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Any     ,	_UpperCamelCase  :					Tuple     ,	_UpperCamelCase  :					Optional[Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Optional[int]     ,	_UpperCamelCase  :					Any     ,	_UpperCamelCase  :					Optional[int])					->str:


          """simple docstring"""


          _lowerCamelCase  :		str			    =    YosoModel(config=_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		Union[str, Any]			    =    model(_UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase)
          _lowerCamelCase  :		Optional[Any]			    =    model(_UpperCamelCase     ,	token_type_ids=_UpperCamelCase)
          _lowerCamelCase  :		Optional[int]			    =    model(_UpperCamelCase)
          self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, self.seq_length, self.hidden_size))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					str     ,	_UpperCamelCase  :					Union[str, Any]     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					Any     ,	_UpperCamelCase  :					int     ,	_UpperCamelCase  :					Optional[Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					List[Any]     ,	_UpperCamelCase  :					Optional[int]     ,	)					->List[str]:


          """simple docstring"""


          _lowerCamelCase  :		Optional[Any]			    =    True
          _lowerCamelCase  :		str			    =    YosoModel(_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		Optional[int]			    =    model(
              _UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	encoder_hidden_states=_UpperCamelCase     ,	encoder_attention_mask=_UpperCamelCase     ,	)
          _lowerCamelCase  :		Union[str, Any]			    =    model(
              _UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	encoder_hidden_states=_UpperCamelCase     ,	)
          _lowerCamelCase  :		List[str]			    =    model(_UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase)
          self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, self.seq_length, self.hidden_size))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Any     ,	_UpperCamelCase  :					int     ,	_UpperCamelCase  :					Optional[int]     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Tuple     ,	_UpperCamelCase  :					Optional[Any]     ,	_UpperCamelCase  :					Optional[Any]     ,	_UpperCamelCase  :					str)					->Tuple:


          """simple docstring"""


          _lowerCamelCase  :		List[Any]			    =    YosoForMaskedLM(config=_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		Tuple			    =    model(_UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	labels=_UpperCamelCase)
          self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.vocab_size))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Union[str, Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					Any     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Optional[int]     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Union[str, Any])					->Any:


          """simple docstring"""


          _lowerCamelCase  :		Tuple			    =    YosoForQuestionAnswering(config=_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		int			    =    model(
              _UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	start_positions=_UpperCamelCase     ,	end_positions=_UpperCamelCase     ,	)
          self.parent.assertEqual(result.start_logits.shape     ,	(self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape     ,	(self.batch_size, self.seq_length))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					Any     ,	_UpperCamelCase  :					Union[str, Any]     ,	_UpperCamelCase  :					Union[str, Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					str)					->Optional[int]:


          """simple docstring"""


          _lowerCamelCase  :		List[Any]			    =    self.num_labels
          _lowerCamelCase  :		Tuple			    =    YosoForSequenceClassification(_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		Optional[Any]			    =    model(_UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	labels=_UpperCamelCase)
          self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_labels))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[Any]     ,	_UpperCamelCase  :					List[str]     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Any     ,	_UpperCamelCase  :					List[str]     ,	_UpperCamelCase  :					Union[str, Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					str)					->List[str]:


          """simple docstring"""


          _lowerCamelCase  :		Tuple			    =    self.num_labels
          _lowerCamelCase  :		Union[str, Any]			    =    YosoForTokenClassification(config=_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		str			    =    model(_UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	labels=_UpperCamelCase)
          self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.num_labels))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[Any]     ,	_UpperCamelCase  :					Tuple     ,	_UpperCamelCase  :					List[str]     ,	_UpperCamelCase  :					Optional[Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					int     ,	_UpperCamelCase  :					Tuple)					->str:


          """simple docstring"""


          _lowerCamelCase  :		Any			    =    self.num_choices
          _lowerCamelCase  :		Tuple			    =    YosoForMultipleChoice(config=_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		Optional[int]			    =    input_ids.unsqueeze(1).expand(-1     ,	self.num_choices     ,	-1).contiguous()
          _lowerCamelCase  :		Union[str, Any]			    =    token_type_ids.unsqueeze(1).expand(-1     ,	self.num_choices     ,	-1).contiguous()
          _lowerCamelCase  :		Optional[Any]			    =    input_mask.unsqueeze(1).expand(-1     ,	self.num_choices     ,	-1).contiguous()
          _lowerCamelCase  :		str			    =    model(
              _UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	labels=_UpperCamelCase     ,	)
          self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_choices))




      def 						_SCREAMING_SNAKE_CASE     (       self  :					Tuple)					->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase  :		Dict			    =    self.prepare_config_and_inputs()
          (
              (
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				
          )  :		Tuple			    =    config_and_inputs
          _lowerCamelCase  :		Union[str, Any]			    =    {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
          return config, inputs_dict





@require_torch
class 						__snake_case						(			__lowerCAmelCase			,	__lowerCAmelCase			,	unittest.TestCase		):





      '''simple docstring'''






      _snake_case     		=							(
          (
              YosoModel,
              YosoForMaskedLM,
              YosoForMultipleChoice,
              YosoForQuestionAnswering,
              YosoForSequenceClassification,
              YosoForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      _snake_case     		=							False
      _snake_case     		=							False
      _snake_case     		=							False

      _snake_case     		=							()
      _snake_case     		=							(
          {
              'feature-extraction': YosoModel,
              'fill-mask': YosoForMaskedLM,
              'question-answering': YosoForQuestionAnswering,
              'text-classification': YosoForSequenceClassification,
              'token-classification': YosoForTokenClassification,
              'zero-shot': YosoForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Optional[int])					->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    YosoModelTester(self)
          _lowerCamelCase  :		Optional[int]			    =    ConfigTester(self     ,	config_class=_UpperCamelCase     ,	hidden_size=37)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Optional[Any])					->Optional[Any]:


          """simple docstring"""


          self.config_tester.run_common_tests()



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Optional[int])					->List[str]:


          """simple docstring"""


          _lowerCamelCase  :		Optional[Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Any)					->Tuple:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
              _lowerCamelCase  :		List[str]			    =    type
              self.model_tester.create_and_check_model(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Optional[Any])					->Optional[int]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Dict)					->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[Any])					->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Tuple)					->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					str)					->Any:


          """simple docstring"""


          _lowerCamelCase  :		List[Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*_UpperCamelCase)



      @slow
      def 						_SCREAMING_SNAKE_CASE     (       self  :					str)					->List[str]:


          """simple docstring"""


          for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _lowerCamelCase  :		Tuple			    =    YosoModel.from_pretrained(_UpperCamelCase)
              self.assertIsNotNone(_UpperCamelCase)




      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[str])					->Union[str, Any]:


          """simple docstring"""


          return





@require_torch
class 						__snake_case						(			unittest.TestCase		):





      '''simple docstring'''






      @slow
      def 						_SCREAMING_SNAKE_CASE     (       self  :					Dict)					->Dict:


          """simple docstring"""


          _lowerCamelCase  :		List[Any]			    =    YosoModel.from_pretrained("""uw-madison/yoso-4096""")
          _lowerCamelCase  :		int			    =    torch.tensor([[0, 1, 2, 3, 4, 5]])

          with torch.no_grad():
              _lowerCamelCase  :		Any			    =    model(_UpperCamelCase)[0]

          _lowerCamelCase  :		List[Any]			    =    torch.Size((1, 6, 768))
          self.assertEqual(output.shape     ,	_UpperCamelCase)

          _lowerCamelCase  :		int			    =    torch.tensor(
              [[[-0.0_6_1_1, 0.1_2_4_2, 0.0_8_4_0], [0.0_2_8_0, -0.0_0_4_8, 0.1_1_2_5], [0.0_1_0_6, 0.0_2_2_6, 0.0_7_5_1]]])

          self.assertTrue(torch.allclose(output[:, :3, :3]     ,	_UpperCamelCase     ,	atol=1E-4))



      @slow
      def 						_SCREAMING_SNAKE_CASE     (       self  :					Union[str, Any])					->str:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096""")
          _lowerCamelCase  :		Tuple			    =    torch.tensor([[0, 1, 2, 3, 4, 5]])

          with torch.no_grad():
              _lowerCamelCase  :		Optional[Any]			    =    model(_UpperCamelCase)[0]

          _lowerCamelCase  :		Union[str, Any]			    =    5_0265

          _lowerCamelCase  :		Any			    =    torch.Size((1, 6, vocab_size))
          self.assertEqual(output.shape     ,	_UpperCamelCase)

          _lowerCamelCase  :		List[str]			    =    torch.tensor(
              [[[-2.1_3_1_3, -3.7_2_8_5, -2.2_4_0_7], [-2.7_0_4_7, -3.3_3_1_4, -2.6_4_0_8], [0.0_6_2_9, -2.5_1_6_6, -0.3_3_5_6]]])

          self.assertTrue(torch.allclose(output[:, :3, :3]     ,	_UpperCamelCase     ,	atol=1E-4))




      @slow
      def 						_SCREAMING_SNAKE_CASE     (       self  :					str)					->Tuple:


          """simple docstring"""


          _lowerCamelCase  :		Tuple			    =    YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096""")
          _lowerCamelCase  :		Any			    =    torch.arange(4096).unsqueeze(0)

          with torch.no_grad():
              _lowerCamelCase  :		Optional[Any]			    =    model(_UpperCamelCase)[0]

          _lowerCamelCase  :		Dict			    =    5_0265

          _lowerCamelCase  :		str			    =    torch.Size((1, 4096, vocab_size))
          self.assertEqual(output.shape     ,	_UpperCamelCase)

          _lowerCamelCase  :		int			    =    torch.tensor(
              [[[-2.3_9_1_4, -4.3_7_4_2, -5.0_9_5_6], [-4.0_9_8_8, -4.2_3_8_4, -7.0_4_0_6], [-3.1_4_2_7, -3.7_1_9_2, -6.6_8_0_0]]])

          self.assertTrue(torch.allclose(output[:, :3, :3]     ,	_UpperCamelCase     ,	atol=1E-4))




