






import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    MODEL_FOR_PRETRAINING_MAPPING,
					    FunnelBaseModel,
					    FunnelForMaskedLM,
					    FunnelForMultipleChoice,
					    FunnelForPreTraining,
					    FunnelForQuestionAnswering,
					    FunnelForSequenceClassification,
					    FunnelForTokenClassification,
					    FunnelModel,
					)


class      __SCREAMING_SNAKE_CASE       :






						def __init__(						self  ,    _UpperCamelCase  ,    _UpperCamelCase=13  ,    _UpperCamelCase=7  ,    _UpperCamelCase=True  ,    _UpperCamelCase=True  ,    _UpperCamelCase=True  ,    _UpperCamelCase=True  ,    _UpperCamelCase=99  ,    _UpperCamelCase=[1, 1, 2]  ,    _UpperCamelCase=1  ,    _UpperCamelCase=32  ,    _UpperCamelCase=4  ,    _UpperCamelCase=8  ,    _UpperCamelCase=37  ,    _UpperCamelCase="gelu_new"  ,    _UpperCamelCase=0.1  ,    _UpperCamelCase=0.1  ,    _UpperCamelCase=0.0  ,    _UpperCamelCase=5_12  ,    _UpperCamelCase=3  ,    _UpperCamelCase=0.02  ,    _UpperCamelCase=3  ,    _UpperCamelCase=4  ,    _UpperCamelCase=None  ,    _UpperCamelCase=False  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    parent
												lowerCAmelCase__         =    batch_size
												lowerCAmelCase__         =    seq_length
												lowerCAmelCase__         =    is_training
												lowerCAmelCase__         =    use_input_mask
												lowerCAmelCase__         =    use_token_type_ids
												lowerCAmelCase__         =    use_labels
												lowerCAmelCase__         =    vocab_size
												lowerCAmelCase__         =    block_sizes
												lowerCAmelCase__         =    num_decoder_layers
												lowerCAmelCase__         =    d_model
												lowerCAmelCase__         =    n_head
												lowerCAmelCase__         =    d_head
												lowerCAmelCase__         =    d_inner
												lowerCAmelCase__         =    hidden_act
												lowerCAmelCase__         =    hidden_dropout
												lowerCAmelCase__         =    attention_dropout
												lowerCAmelCase__         =    activation_dropout
												lowerCAmelCase__         =    max_position_embeddings
												lowerCAmelCase__         =    type_vocab_size
												lowerCAmelCase__         =    2
												lowerCAmelCase__         =    num_labels
												lowerCAmelCase__         =    num_choices
												lowerCAmelCase__         =    scope
												lowerCAmelCase__         =    initializer_std

												# Used in the tests to check the size of the first attention layer
												lowerCAmelCase__         =    n_head
												# Used in the tests to check the size of the first hidden state
												lowerCAmelCase__         =    self.d_model
												# Used in the tests to check the number of output hidden states/attentions
												lowerCAmelCase__         =    sum(self.block_sizes ) + (0 if base else self.num_decoder_layers)
												# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
												# the last hidden state of the first block (which is the first hidden state of the decoder).
												if not base:
																		lowerCAmelCase__         =    self.num_hidden_layers + 2






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    ids_tensor([self.batch_size, self.seq_length]  ,    self.vocab_size )

												lowerCAmelCase__         =    None
												if self.use_input_mask:
																		lowerCAmelCase__         =    ids_tensor([self.batch_size, self.seq_length]  ,    vocab_size=2 )

												lowerCAmelCase__         =    None
												if self.use_token_type_ids:
																		lowerCAmelCase__         =    ids_tensor([self.batch_size, self.seq_length]  ,    self.type_vocab_size )

												lowerCAmelCase__         =    None
												lowerCAmelCase__         =    None
												lowerCAmelCase__         =    None
												if self.use_labels:
																		lowerCAmelCase__         =    ids_tensor([self.batch_size]  ,    self.type_sequence_label_size )
																		lowerCAmelCase__         =    ids_tensor([self.batch_size, self.seq_length]  ,    self.num_labels )
																		lowerCAmelCase__         =    ids_tensor([self.batch_size]  ,    self.num_choices )
																		lowerCAmelCase__         =    ids_tensor([self.batch_size, self.seq_length]  ,    1 )

												lowerCAmelCase__         =    self.get_config()

												return (
												    config,
												    input_ids,
												    token_type_ids,
												    input_mask,
												    sequence_labels,
												    token_labels,
												    choice_labels,
												    fake_token_labels,
												)






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												return FunnelConfig(
												    vocab_size=self.vocab_size  ,    block_sizes=self.block_sizes  ,    num_decoder_layers=self.num_decoder_layers  ,    d_model=self.d_model  ,    n_head=self.n_head  ,    d_head=self.d_head  ,    d_inner=self.d_inner  ,    hidden_act=self.hidden_act  ,    hidden_dropout=self.hidden_dropout  ,    attention_dropout=self.attention_dropout  ,    activation_dropout=self.activation_dropout  ,    max_position_embeddings=self.max_position_embeddings  ,    type_vocab_size=self.type_vocab_size  ,    initializer_std=self.initializer_std  ,    )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    FunnelModel(config=_UpperCamelCase )
												model.to(_UpperCamelCase )
												model.eval()
												lowerCAmelCase__         =    model(_UpperCamelCase  ,    attention_mask=_UpperCamelCase  ,    token_type_ids=_UpperCamelCase )
												lowerCAmelCase__         =    model(_UpperCamelCase  ,    token_type_ids=_UpperCamelCase )
												lowerCAmelCase__         =    model(_UpperCamelCase )
												self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.d_model) )

												lowerCAmelCase__         =    False
												lowerCAmelCase__         =    model(_UpperCamelCase )
												self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.d_model) )

												lowerCAmelCase__         =    False
												lowerCAmelCase__         =    model(_UpperCamelCase )
												self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.d_model) )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    FunnelBaseModel(config=_UpperCamelCase )
												model.to(_UpperCamelCase )
												model.eval()
												lowerCAmelCase__         =    model(_UpperCamelCase  ,    attention_mask=_UpperCamelCase  ,    token_type_ids=_UpperCamelCase )
												lowerCAmelCase__         =    model(_UpperCamelCase  ,    token_type_ids=_UpperCamelCase )
												lowerCAmelCase__         =    model(_UpperCamelCase )
												self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, 2, self.d_model) )

												lowerCAmelCase__         =    False
												lowerCAmelCase__         =    model(_UpperCamelCase )
												self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, 3, self.d_model) )

												lowerCAmelCase__         =    False
												lowerCAmelCase__         =    model(_UpperCamelCase )
												self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, 2, self.d_model) )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.num_labels
												lowerCAmelCase__         =    FunnelForPreTraining(config=_UpperCamelCase )
												model.to(_UpperCamelCase )
												model.eval()
												lowerCAmelCase__         =    model(_UpperCamelCase  ,    attention_mask=_UpperCamelCase  ,    token_type_ids=_UpperCamelCase  ,    labels=_UpperCamelCase )
												self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length) )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    FunnelForMaskedLM(config=_UpperCamelCase )
												model.to(_UpperCamelCase )
												model.eval()
												lowerCAmelCase__         =    model(_UpperCamelCase  ,    attention_mask=_UpperCamelCase  ,    token_type_ids=_UpperCamelCase  ,    labels=_UpperCamelCase )
												self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.vocab_size) )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.num_labels
												lowerCAmelCase__         =    FunnelForSequenceClassification(_UpperCamelCase )
												model.to(_UpperCamelCase )
												model.eval()
												lowerCAmelCase__         =    model(_UpperCamelCase  ,    attention_mask=_UpperCamelCase  ,    token_type_ids=_UpperCamelCase  ,    labels=_UpperCamelCase )
												self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.num_labels) )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.num_choices
												lowerCAmelCase__         =    FunnelForMultipleChoice(config=_UpperCamelCase )
												model.to(_UpperCamelCase )
												model.eval()
												lowerCAmelCase__         =    input_ids.unsqueeze(1 ).expand(-1  ,    self.num_choices  ,    -1 ).contiguous()
												lowerCAmelCase__         =    token_type_ids.unsqueeze(1 ).expand(-1  ,    self.num_choices  ,    -1 ).contiguous()
												lowerCAmelCase__         =    input_mask.unsqueeze(1 ).expand(-1  ,    self.num_choices  ,    -1 ).contiguous()
												lowerCAmelCase__         =    model(
												    _UpperCamelCase  ,    attention_mask=_UpperCamelCase  ,    token_type_ids=_UpperCamelCase  ,    labels=_UpperCamelCase  ,    )
												self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.num_choices) )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.num_labels
												lowerCAmelCase__         =    FunnelForTokenClassification(config=_UpperCamelCase )
												model.to(_UpperCamelCase )
												model.eval()
												lowerCAmelCase__         =    model(_UpperCamelCase  ,    attention_mask=_UpperCamelCase  ,    token_type_ids=_UpperCamelCase  ,    labels=_UpperCamelCase )
												self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.num_labels) )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    FunnelForQuestionAnswering(config=_UpperCamelCase )
												model.to(_UpperCamelCase )
												model.eval()
												lowerCAmelCase__         =    model(
												    _UpperCamelCase  ,    attention_mask=_UpperCamelCase  ,    token_type_ids=_UpperCamelCase  ,    start_positions=_UpperCamelCase  ,    end_positions=_UpperCamelCase  ,    )
												self.parent.assertEqual(result.start_logits.shape  ,    (self.batch_size, self.seq_length) )
												self.parent.assertEqual(result.end_logits.shape  ,    (self.batch_size, self.seq_length) )







						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.prepare_config_and_inputs()
												(
												    (
												    lowerCAmelCase__
												)    ,							(
												    lowerCAmelCase__
												)    ,							(
												    lowerCAmelCase__
												)    ,							(
												    lowerCAmelCase__
												)    ,							(
												    lowerCAmelCase__
												)    ,							(
												    lowerCAmelCase__
												)    ,							(
												    lowerCAmelCase__
												)    ,							(
												    lowerCAmelCase__
												)    ,							
												)         =    config_and_inputs
												lowerCAmelCase__         =    {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
												return config, inputs_dict




@require_torch
class      __SCREAMING_SNAKE_CASE       ( __lowercase		,       __lowercase		,       unittest.TestCase):
						_SCREAMING_SNAKE_CASE						:							List[Any]				=       False
						_SCREAMING_SNAKE_CASE						:							List[Any]				=       False
						_SCREAMING_SNAKE_CASE						:							List[str]				=       (
						    (
						        FunnelModel,
						        FunnelForMaskedLM,
						        FunnelForPreTraining,
						        FunnelForQuestionAnswering,
						        FunnelForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						_SCREAMING_SNAKE_CASE						:							Dict				=       (
						    {
						        '''feature-extraction''': (FunnelBaseModel, FunnelModel),
						        '''fill-mask''': FunnelForMaskedLM,
						        '''question-answering''': FunnelForQuestionAnswering,
						        '''text-classification''': FunnelForSequenceClassification,
						        '''token-classification''': FunnelForTokenClassification,
						        '''zero-shot''': FunnelForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase=False ):






												"""simple docstring"""
												lowerCAmelCase__         =    super()._prepare_for_class(_UpperCamelCase  ,    _UpperCamelCase  ,    return_labels=_UpperCamelCase )

												if return_labels:
																		if model_class in get_values(_UpperCamelCase ):
																								lowerCAmelCase__         =    torch.zeros(
																								    (self.model_tester.batch_size, self.model_tester.seq_length)  ,    dtype=torch.long  ,    device=_UpperCamelCase )
												return inputs_dict






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    FunnelModelTester(self )
												lowerCAmelCase__         =    ConfigTester(self  ,    config_class=_UpperCamelCase )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												self.config_tester.run_common_tests()






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*_UpperCamelCase )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_pretraining(*_UpperCamelCase )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*_UpperCamelCase )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*_UpperCamelCase )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*_UpperCamelCase )







						def    UpperCamelCase__   (						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												if hasattr(_UpperCamelCase  ,    'weight' ) and module.weight is not None:
																		module.weight.data.fill_(3 )
												if hasattr(_UpperCamelCase  ,    'bias' ) and module.bias is not None:
																		module.bias.data.fill_(3 )

												for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																		if hasattr(_UpperCamelCase  ,    _UpperCamelCase ) and getattr(_UpperCamelCase  ,    _UpperCamelCase ) is not None:
																								lowerCAmelCase__         =    getattr(_UpperCamelCase  ,    _UpperCamelCase )
																								weight.data.fill_(3 )




@require_torch
class      __SCREAMING_SNAKE_CASE       ( __lowercase		,       unittest.TestCase):
						_SCREAMING_SNAKE_CASE						:							str				=       False
						_SCREAMING_SNAKE_CASE						:							Optional[Any]				=       False
						_SCREAMING_SNAKE_CASE						:							str				=       (
						    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
						)






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    FunnelModelTester(self  ,    base=_UpperCamelCase )
												lowerCAmelCase__         =    ConfigTester(self  ,    config_class=_UpperCamelCase )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												self.config_tester.run_common_tests()






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_base_model(*_UpperCamelCase )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*_UpperCamelCase )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_multiple_choice(*_UpperCamelCase )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__    ,							lowerCAmelCase__         =    self.model_tester.prepare_config_and_inputs_for_common()
												lowerCAmelCase__         =    True

												for model_class in self.all_model_classes:
																		if model_class.__name__ == "FunnelBaseModel":
																								continue
																		lowerCAmelCase__         =    model_class(_UpperCamelCase )
																		model.to(_UpperCamelCase )
																		model.train()
																		lowerCAmelCase__         =    self._prepare_for_class(_UpperCamelCase  ,    _UpperCamelCase  ,    return_labels=_UpperCamelCase )
																		lowerCAmelCase__         =    model(**_UpperCamelCase ).loss
																		loss.backward()







						def    UpperCamelCase__   (						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												if hasattr(_UpperCamelCase  ,    'weight' ) and module.weight is not None:
																		module.weight.data.fill_(3 )
												if hasattr(_UpperCamelCase  ,    'bias' ) and module.bias is not None:
																		module.bias.data.fill_(3 )

												for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																		if hasattr(_UpperCamelCase  ,    _UpperCamelCase ) and getattr(_UpperCamelCase  ,    _UpperCamelCase ) is not None:
																								lowerCAmelCase__         =    getattr(_UpperCamelCase  ,    _UpperCamelCase )
																								weight.data.fill_(3 )




@require_torch
@require_sentencepiece
@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    13
												lowerCAmelCase__         =    7
												lowerCAmelCase__         =    torch.arange(0  ,    batch_size * sequence_length ).long().reshape(_UpperCamelCase  ,    _UpperCamelCase )
												lowerCAmelCase__         =    [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
												lowerCAmelCase__         =    torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths] )

												lowerCAmelCase__         =    FunnelModel.from_pretrained('sgugger/funnel-random-tiny' )
												lowerCAmelCase__         =    model(_UpperCamelCase  ,    token_type_ids=_UpperCamelCase )[0].abs()

												lowerCAmelCase__         =    torch.tensor(23_44.83_52 )
												lowerCAmelCase__         =    torch.tensor(0.80_52 )
												self.assertTrue(torch.allclose(output.sum()  ,    _UpperCamelCase  ,    atol=1E-4 ) )
												self.assertTrue(torch.allclose(output.mean()  ,    _UpperCamelCase  ,    atol=1E-4 ) )

												lowerCAmelCase__         =    torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]] )
												lowerCAmelCase__         =    model(_UpperCamelCase  ,    attention_mask=_UpperCamelCase  ,    token_type_ids=_UpperCamelCase )[0].abs()

												lowerCAmelCase__         =    torch.tensor(23_43.84_25 )
												lowerCAmelCase__         =    torch.tensor(0.80_49 )
												self.assertTrue(torch.allclose(output.sum()  ,    _UpperCamelCase  ,    atol=1E-4 ) )
												self.assertTrue(torch.allclose(output.mean()  ,    _UpperCamelCase  ,    atol=1E-4 ) )







						@slow
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    FunnelTokenizer.from_pretrained('huggingface/funnel-small' )
												lowerCAmelCase__         =    FunnelModel.from_pretrained('huggingface/funnel-small' )
												lowerCAmelCase__         =    tokenizer('Hello! I am the Funnel Transformer model.'  ,    return_tensors='pt' )
												lowerCAmelCase__         =    model(**_UpperCamelCase )[0]

												lowerCAmelCase__         =    torch.tensor(2_35.72_46 )
												lowerCAmelCase__         =    torch.tensor(0.02_56 )
												self.assertTrue(torch.allclose(output.sum()  ,    _UpperCamelCase  ,    atol=1E-4 ) )
												self.assertTrue(torch.allclose(output.mean()  ,    _UpperCamelCase  ,    atol=1E-4 ) )













import importlib
from typing import List, Optional, Union

from seqeval.metrics import accuracy_score, classification_report

import datasets


__snake_case     :     Optional[int]       =  """\
@inproceedings{ramshaw-marcus-1995-text,
    title = \"Text Chunking using Transformation-Based Learning\",
    author = \"Ramshaw, Lance  and
      Marcus, Mitch\",
    booktitle = \"Third Workshop on Very Large Corpora\",
    year = \"1995\",
    url = \"https://www.aclweb.org/anthology/W95-0107\",
}
@misc{seqeval,
  title={{seqeval}: A Python framework for sequence labeling evaluation},
  url={https://github.com/chakki-works/seqeval},
  note={Software available from https://github.com/chakki-works/seqeval},
  author={Hiroki Nakayama},
  year={2018},
}
"""

__snake_case     :     Any       =  """\
seqeval is a Python framework for sequence labeling evaluation.
seqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on.

This is well-tested by using the Perl script conlleval, which can be used for
measuring the performance of a system that has processed the CoNLL-2000 shared task data.

seqeval supports following formats:
IOB1
IOB2
IOE1
IOE2
IOBES

See the [README.md] file at https://github.com/chakki-works/seqeval for more information.
"""

__snake_case     :     Optional[int]       =  """
Produces labelling scores along with its sufficient statistics
from a source against one or more references.

Args:
    predictions: List of List of predicted labels (Estimated targets as returned by a tagger)
    references: List of List of reference labels (Ground truth (correct) target values)
    suffix: True if the IOB prefix is after type, False otherwise. default: False
    scheme: Specify target tagging scheme. Should be one of [\"IOB1\", \"IOB2\", \"IOE1\", \"IOE2\", \"IOBES\", \"BILOU\"].
        default: None
    mode: Whether to count correct entity labels with incorrect I/B tags as true positives or not.
        If you want to only count exact matches, pass mode=\"strict\". default: None.
    sample_weight: Array-like of shape (n_samples,), weights for individual samples. default: None
    zero_division: Which value to substitute as a metric value when encountering zero division. Should be on of 0, 1,
        \"warn\". \"warn\" acts as 0, but the warning is raised.

Returns:
    'scores': dict. Summary of the scores for overall and per type
        Overall:
            'accuracy': accuracy,
            'precision': precision,
            'recall': recall,
            'f1': F1 score, also known as balanced F-score or F-measure,
        Per type:
            'precision': precision,
            'recall': recall,
            'f1': F1 score, also known as balanced F-score or F-measure
Examples:

    >>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]
    >>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]
    >>> seqeval = datasets.load_metric(\"seqeval\")
    >>> results = seqeval.compute(predictions=predictions, references=references)
    >>> print(list(results.keys()))
    ['MISC', 'PER', 'overall_precision', 'overall_recall', 'overall_f1', 'overall_accuracy']
    >>> print(results[\"overall_f1\"])
    0.5
    >>> print(results[\"PER\"][\"f1\"])
    1.0
"""


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,       _KWARGS_DESCRIPTION)
class      __SCREAMING_SNAKE_CASE       ( datasets.Metric):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												return datasets.MetricInfo(
												    description=_DESCRIPTION  ,    citation=_CITATION  ,    homepage='https://github.com/chakki-works/seqeval'  ,    inputs_description=_KWARGS_DESCRIPTION  ,    features=datasets.Features(
												        {
												            'predictions': datasets.Sequence(datasets.Value('string'  ,    id='label' )  ,    id='sequence' ),
												            'references': datasets.Sequence(datasets.Value('string'  ,    id='label' )  ,    id='sequence' ),
												        } )  ,    codebase_urls=['https://github.com/chakki-works/seqeval']  ,    reference_urls=['https://github.com/chakki-works/seqeval']  ,    )







						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase = False  ,    _UpperCamelCase = None  ,    _UpperCamelCase = None  ,    _UpperCamelCase = None  ,    _UpperCamelCase = "warn"  ,    ):






												"""simple docstring"""
												if scheme is not None:
																		try:
																								lowerCAmelCase__         =    importlib.import_module('seqeval.scheme' )
																								lowerCAmelCase__         =    getattr(_UpperCamelCase  ,    _UpperCamelCase )
																		except AttributeError:
																								raise ValueError(F"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {scheme}" )
												lowerCAmelCase__         =    classification_report(
												    y_true=_UpperCamelCase  ,    y_pred=_UpperCamelCase  ,    suffix=_UpperCamelCase  ,    output_dict=_UpperCamelCase  ,    scheme=_UpperCamelCase  ,    mode=_UpperCamelCase  ,    sample_weight=_UpperCamelCase  ,    zero_division=_UpperCamelCase  ,    )
												report.pop('macro avg' )
												report.pop('weighted avg' )
												lowerCAmelCase__         =    report.pop('micro avg' )

												lowerCAmelCase__         =    {
												    type_name: {
												        'precision': score['precision'],
												        'recall': score['recall'],
												        'f1': score['f1-score'],
												        'number': score['support'],
												    }
												    for type_name, score in report.items()
												}
												lowerCAmelCase__         =    overall_score['precision']
												lowerCAmelCase__         =    overall_score['recall']
												lowerCAmelCase__         =    overall_score['f1-score']
												lowerCAmelCase__         =    accuracy_score(y_true=_UpperCamelCase  ,    y_pred=_UpperCamelCase )

												return scores





