



from __future__ import annotations

import unittest

from transformers import DebertaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import (
     TFDebertaForMaskedLM,
     TFDebertaForQuestionAnswering,
     TFDebertaForSequenceClassification,
     TFDebertaForTokenClassification,
     TFDebertaModel,
 )
class        UpperCAmelCase__  :



 def __init__(  self		,	lowercase		,	lowercase=1_3		,	lowercase=7		,	lowercase=True		,	lowercase=True		,	lowercase=True		,	lowercase=True		,	lowercase=9_9		,	lowercase=3_2		,	lowercase=2		,	lowercase=4		,	lowercase=3_7		,	lowercase="gelu"		,	lowercase=0.1		,	lowercase=0.1		,	lowercase=5_1_2		,	lowercase=1_6		,	lowercase=2		,	lowercase=0.02		,	lowercase=3		,	lowercase=4		,	lowercase=None		,	)  ->							List[str]:
  __UpperCamelCase				      =  parent
  __UpperCamelCase				      =  1_3
  __UpperCamelCase				      =  7
  __UpperCamelCase				      =  True
  __UpperCamelCase				      =  True
  __UpperCamelCase				      =  True
  __UpperCamelCase				      =  True
  __UpperCamelCase				      =  9_9
  __UpperCamelCase				      =  3_2
  __UpperCamelCase				      =  2
  __UpperCamelCase				      =  4
  __UpperCamelCase				      =  3_7
  __UpperCamelCase				      =  """gelu"""
  __UpperCamelCase				      =  0.1
  __UpperCamelCase				      =  0.1
  __UpperCamelCase				      =  5_1_2
  __UpperCamelCase				      =  1_6
  __UpperCamelCase				      =  False
  __UpperCamelCase				      =  -1
  __UpperCamelCase				      =  True
  __UpperCamelCase				      =  2
  __UpperCamelCase				      =  0.02
  __UpperCamelCase				      =  3
  __UpperCamelCase				      =  4
  __UpperCamelCase				      =  None



 def    __lowerCamelCase				(  self			)  ->							Union[str, Any]:
  __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.vocab_size			)

  __UpperCamelCase				      =  None
  if self.use_input_mask:
   __UpperCamelCase				      =  random_attention_mask([self.batch_size, self.seq_length]			)

  __UpperCamelCase				      =  None
  if self.use_token_type_ids:
   __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.type_vocab_size			)

  __UpperCamelCase				      =  None
  __UpperCamelCase				      =  None
  __UpperCamelCase				      =  None
  if self.use_labels:
   __UpperCamelCase				      =  ids_tensor([self.batch_size]		,	self.type_sequence_label_size			)
   __UpperCamelCase				      =  ids_tensor([self.batch_size, self.seq_length]		,	self.num_labels			)

  __UpperCamelCase				      =  DebertaConfig(
      vocab_size=self.vocab_size		,	hidden_size=self.hidden_size		,	num_hidden_layers=self.num_hidden_layers		,	num_attention_heads=self.num_attention_heads		,	intermediate_size=self.intermediate_size		,	hidden_act=self.hidden_act		,	hidden_dropout_prob=self.hidden_dropout_prob		,	attention_probs_dropout_prob=self.attention_probs_dropout_prob		,	max_position_embeddings=self.max_position_embeddings		,	type_vocab_size=self.type_vocab_size		,	relative_attention=self.relative_attention		,	max_relative_positions=self.max_relative_positions		,	position_biased_input=self.position_biased_input		,	initializer_range=self.initializer_range		,	return_dict=lowercase		,	)

  return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase			)  ->							str:
  __UpperCamelCase				      =  TFDebertaModel(config=lowercase			)
  __UpperCamelCase				      =  {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

  __UpperCamelCase				      =  [input_ids, input_mask]
  __UpperCamelCase				      =  model(lowercase			)

  __UpperCamelCase				      =  model(lowercase			)

  self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.hidden_size)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase			)  ->							List[str]:
  __UpperCamelCase				      =  TFDebertaForMaskedLM(config=lowercase			)
  __UpperCamelCase				      =  {
      """input_ids""": input_ids,
      """attention_mask""": input_mask,
      """token_type_ids""": token_type_ids,
  }
  __UpperCamelCase				      =  model(lowercase			)
  self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.vocab_size)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase			)  ->							Optional[Any]:
  __UpperCamelCase				      =  self.num_labels
  __UpperCamelCase				      =  TFDebertaForSequenceClassification(config=lowercase			)
  __UpperCamelCase				      =  {
      """input_ids""": input_ids,
      """attention_mask""": input_mask,
      """token_type_ids""": token_type_ids,
  }

  __UpperCamelCase				      =  model(lowercase			)
  self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.num_labels)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase			)  ->							Dict:
  __UpperCamelCase				      =  self.num_labels
  __UpperCamelCase				      =  TFDebertaForTokenClassification(config=lowercase			)
  __UpperCamelCase				      =  {
      """input_ids""": input_ids,
      """attention_mask""": input_mask,
      """token_type_ids""": token_type_ids,
  }
  __UpperCamelCase				      =  model(lowercase			)
  self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.num_labels)			)



 def    __lowerCamelCase				(  self		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase		,	lowercase			)  ->							str:
  __UpperCamelCase				      =  TFDebertaForQuestionAnswering(config=lowercase			)
  __UpperCamelCase				      =  {
      """input_ids""": input_ids,
      """attention_mask""": input_mask,
      """token_type_ids""": token_type_ids,
  }

  __UpperCamelCase				      =  model(lowercase			)
  self.parent.assertEqual(result.start_logits.shape		,	(self.batch_size, self.seq_length)			)
  self.parent.assertEqual(result.end_logits.shape		,	(self.batch_size, self.seq_length)			)




 def    __lowerCamelCase				(  self			)  ->							Dict:
  __UpperCamelCase				      =  self.prepare_config_and_inputs()
  (
      (
      __UpperCamelCase		
  )      ,		(
      __UpperCamelCase		
  )      ,		(
      __UpperCamelCase		
  )      ,		(
      __UpperCamelCase		
  )      ,		(
      __UpperCamelCase		
  )      ,		(
      __UpperCamelCase		
  )      ,		(
      __UpperCamelCase		
  )      ,				
  )				      =  config_and_inputs
  __UpperCamelCase				      =  {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
  return config, inputs_dict


@require_tf
class        UpperCAmelCase__  (    UpperCAmelCase_							,			UpperCAmelCase_							,			unittest.TestCase):
 __SCREAMING_SNAKE_CASE					      =				(
     (
         TFDebertaModel,
         TFDebertaForMaskedLM,
         TFDebertaForQuestionAnswering,
         TFDebertaForSequenceClassification,
         TFDebertaForTokenClassification,
     )
     if is_tf_available()
     else ()
 )
 __SCREAMING_SNAKE_CASE					      =				(
     {
         '''feature-extraction''': TFDebertaModel,
         '''fill-mask''': TFDebertaForMaskedLM,
         '''question-answering''': TFDebertaForQuestionAnswering,
         '''text-classification''': TFDebertaForSequenceClassification,
         '''token-classification''': TFDebertaForTokenClassification,
         '''zero-shot''': TFDebertaForSequenceClassification,
     }
     if is_tf_available()
     else {}
 )

 __SCREAMING_SNAKE_CASE					      =				False
 __SCREAMING_SNAKE_CASE					      =				False



 def    __lowerCamelCase				(  self			)  ->							Dict:
  __UpperCamelCase				      =  TFDebertaModelTester(self			)
  __UpperCamelCase				      =  ConfigTester(self		,	config_class=lowercase		,	hidden_size=3_7			)



 def    __lowerCamelCase				(  self			)  ->							Union[str, Any]:
  self.config_tester.run_common_tests()



 def    __lowerCamelCase				(  self			)  ->							Optional[Any]:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_model(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							int:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_masked_lm(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							str:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_question_answering(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							Union[str, Any]:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_sequence_classification(*lowercase			)



 def    __lowerCamelCase				(  self			)  ->							Optional[Any]:
  __UpperCamelCase				      =  self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_token_classification(*lowercase			)




 @slow
 def    __lowerCamelCase				(  self			)  ->							Dict:
  __UpperCamelCase				      =  TFDebertaModel.from_pretrained("""kamalkraj/deberta-base"""			)
  self.assertIsNotNone(lowercase			)


@require_tf
class        UpperCAmelCase__  (    unittest.TestCase):



 @unittest.skip(reason="""Model not available yet"""			)
 def    __lowerCamelCase				(  self			)  ->							int:
  pass




 @slow
 def    __lowerCamelCase				(  self			)  ->							Optional[int]:
  __UpperCamelCase				      =  TFDebertaModel.from_pretrained("""kamalkraj/deberta-base"""			)
  __UpperCamelCase				      =  tf.constant([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]			)
  __UpperCamelCase				      =  tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]			)
  __UpperCamelCase				      =  model(lowercase		,	attention_mask=lowercase			)[0]

  __UpperCamelCase				      =  tf.constant(
      [
          [
              [-0.59_855_896, -0.80_552_566, -0.8_462_135],
              [1.4_484_025, -0.93_483_794, -0.80_593_085],
              [0.3_122_741, 0.00_316_059, -1.4_131_377],
          ]
      ]			)
  tf.debugging.assert_near(output[:, 1:4, 1:4]		,	lowercase		,	atol=1E-4			)










'''simple docstring'''



import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Image
from datasets.packaged_modules.text.text import Text

from ..utils import require_pil







@pytest.fixture
def _lowercase   (  __A       ):



 '''simple docstring'''



 __UpperCamelCase				      =  tmp_path / """text.txt"""
 __UpperCamelCase				      =  textwrap.dedent(
     """\
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Second paragraph:
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
        """       )
 with open(__A							,"""w"""							,encoding="""utf-8"""       ) as f:
  f.write(__A       )
 return str(__A       )







@pytest.fixture
def _lowercase   (  __A							,__A       ):



 '''simple docstring'''



 __UpperCamelCase				      =  tmp_path / """text_with_image.txt"""
 with open(__A							,"""w"""							,encoding="""utf-8"""       ) as f:
  f.write(__A       )
 return str(__A       )







@pytest.mark.parametrize("""keep_linebreaks"""							,[True, False]       )
def _lowercase   (  __A							,__A       ):



 '''simple docstring'''



 with open(__A							,encoding="""utf-8"""       ) as f:
  __UpperCamelCase				      =  f.read().splitlines(keepends=__A       )
 __UpperCamelCase				      =  Text(keep_linebreaks=__A							,encoding="""utf-8"""       )
 __UpperCamelCase				      =  text._generate_tables([[text_file]]       )
 __UpperCamelCase				      =  pa.concat_tables([table for _, table in generator]       ).to_pydict()["""text"""]
 assert generated_content == expected_content







@require_pil
def _lowercase   (  __A       ):



 '''simple docstring'''



 with open(__A							,encoding="""utf-8"""       ) as f:
  __UpperCamelCase				      =  f.read().splitlines()[0]
 __UpperCamelCase				      =  Text(encoding="""utf-8"""							,features=Features({"""image""": Image()}       )       )
 __UpperCamelCase				      =  text._generate_tables([[text_file_with_image]]       )
 __UpperCamelCase				      =  pa.concat_tables([table for _, table in generator]       )
 assert pa_table.schema.field("""image"""       ).type == Image()()
 __UpperCamelCase				      =  pa_table.to_pydict()["""image"""]
 assert generated_content == [{"path": image_file, "bytes": None}]







@pytest.mark.parametrize("""sample_by"""							,["""line""", """paragraph""", """document"""]       )
def _lowercase   (  __A							,__A       ):



 '''simple docstring'''



 with open(__A							,encoding="""utf-8"""       ) as f:
  __UpperCamelCase				      =  f.read()
 if sample_by == "line":
  __UpperCamelCase				      =  expected_content.splitlines()
 elif sample_by == "paragraph":
  __UpperCamelCase				      =  expected_content.split("""\n\n"""       )
 elif sample_by == "document":
  __UpperCamelCase				      =  [expected_content]
 __UpperCamelCase				      =  Text(sample_by=__A							,encoding="""utf-8"""							,chunksize=100       )
 __UpperCamelCase				      =  text._generate_tables([[text_file]]       )
 __UpperCamelCase				      =  pa.concat_tables([table for _, table in generator]       ).to_pydict()["""text"""]
 assert generated_content == expected_content





