





"""simple docstring"""






import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin



class 						lowerCAmelCase       (   snake_case				):
       lowerCAmelCase__    			=	["""image_processor""", """tokenizer"""]
       lowerCAmelCase__    			=	"""AutoImageProcessor"""
       lowerCAmelCase__    			=	"""AutoTokenizer"""






       def __init__( self     ,	a__=None     ,	a__=None     ,	**a__  ):
              _UpperCAmelCase			     =  None
              if "feature_extractor" in kwargs:
                     warnings.warn(
                         'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
                         ' instead.'     ,	a__     ,	)
                     _UpperCAmelCase			     =  kwargs.pop('feature_extractor'  )

              _UpperCAmelCase			     =  image_processor if image_processor is not None else feature_extractor
              if image_processor is None:
                     raise ValueError('You need to specify an `image_processor`.'  )
              if tokenizer is None:
                     raise ValueError('You need to specify a `tokenizer`.'  )

              super().__init__(a__     ,	a__  )
              _UpperCAmelCase			     =  self.image_processor
              _UpperCAmelCase			     =  False






       def __call__( self     ,	*a__     ,	**a__  ):
              # For backward compatibility
              if self._in_target_context_manager:
                     return self.current_processor(*a__     ,	**a__  )

              _UpperCAmelCase			     =  kwargs.pop('images'     ,	a__  )
              _UpperCAmelCase			     =  kwargs.pop('text'     ,	a__  )
              if len(a__  ) > 0:
                     _UpperCAmelCase			     =  args[0]
                     _UpperCAmelCase			     =  args[1:]

              if images is None and text is None:
                     raise ValueError('You need to specify either an `images` or `text` input to process.'  )

              if images is not None:
                     _UpperCAmelCase			     =  self.image_processor(a__     ,	*a__     ,	**a__  )
              if text is not None:
                     _UpperCAmelCase			     =  self.tokenizer(a__     ,	**a__  )

              if text is None:
                     return inputs
              elif images is None:
                     return encodings
              else:
                     _UpperCAmelCase			     =  encodings['input_ids']
                     return inputs






       def 				__A			( self     ,	*a__     ,	**a__  ):
              return self.tokenizer.batch_decode(*a__     ,	**a__  )






       def 				__A			( self     ,	*a__     ,	**a__  ):
              return self.tokenizer.decode(*a__     ,	**a__  )






       @contextmanager
       def 				__A			( self  ):
              warnings.warn(
                  '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
                  'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
                  'your images inputs, or in a separate call.'  )
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  self.tokenizer
              yield
              _UpperCAmelCase			     =  self.image_processor
              _UpperCAmelCase			     =  False






       @property
       def 				__A			( self  ):
              warnings.warn(
                  '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'     ,	a__     ,	)
              return self.image_processor_class






       @property
       def 				__A			( self  ):
              warnings.warn(
                  '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'     ,	a__     ,	)
              return self.image_processor













"""simple docstring"""






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMModel,
        TFXLMWithLMHeadModel,
        XLMConfig,
    )



class 						lowerCAmelCase       :






       def __init__( self     ,	a__     ,	):
              _UpperCAmelCase			     =  parent
              _UpperCAmelCase			     =  13
              _UpperCAmelCase			     =  7
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  False
              _UpperCAmelCase			     =  False
              _UpperCAmelCase			     =  False
              _UpperCAmelCase			     =  2
              _UpperCAmelCase			     =  99
              _UpperCAmelCase			     =  0
              _UpperCAmelCase			     =  32
              _UpperCAmelCase			     =  2
              _UpperCAmelCase			     =  4
              _UpperCAmelCase			     =  0.1
              _UpperCAmelCase			     =  0.1
              _UpperCAmelCase			     =  5_12
              _UpperCAmelCase			     =  16
              _UpperCAmelCase			     =  2
              _UpperCAmelCase			     =  0.02
              _UpperCAmelCase			     =  3
              _UpperCAmelCase			     =  4
              _UpperCAmelCase			     =  'last'
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  None
              _UpperCAmelCase			     =  0






       def 				__A			( self  ):
              _UpperCAmelCase			     =  ids_tensor([self.batch_size, self.seq_length]     ,	self.vocab_size  )
              _UpperCAmelCase			     =  random_attention_mask([self.batch_size, self.seq_length]     ,	dtype=tf.floataa  )

              _UpperCAmelCase			     =  None
              if self.use_input_lengths:
                     _UpperCAmelCase			     =  (
                         ids_tensor([self.batch_size]     ,	vocab_size=2  ) + self.seq_length - 2
                     )  # small variation of seq_length

              _UpperCAmelCase			     =  None
              if self.use_token_type_ids:
                     _UpperCAmelCase			     =  ids_tensor([self.batch_size, self.seq_length]     ,	self.n_langs  )

              _UpperCAmelCase			     =  None
              _UpperCAmelCase			     =  None
              _UpperCAmelCase			     =  None
              if self.use_labels:
                     _UpperCAmelCase			     =  ids_tensor([self.batch_size]     ,	self.type_sequence_label_size  )
                     _UpperCAmelCase			     =  ids_tensor([self.batch_size, self.seq_length]     ,	self.num_labels  )
                     _UpperCAmelCase			     =  ids_tensor([self.batch_size]     ,	2     ,	dtype=tf.floataa  )
                     _UpperCAmelCase			     =  ids_tensor([self.batch_size]     ,	self.num_choices  )

              _UpperCAmelCase			     =  XLMConfig(
                  vocab_size=self.vocab_size     ,	n_special=self.n_special     ,	emb_dim=self.hidden_size     ,	n_layers=self.num_hidden_layers     ,	n_heads=self.num_attention_heads     ,	dropout=self.hidden_dropout_prob     ,	attention_dropout=self.attention_probs_dropout_prob     ,	gelu_activation=self.gelu_activation     ,	sinusoidal_embeddings=self.sinusoidal_embeddings     ,	asm=self.asm     ,	causal=self.causal     ,	n_langs=self.n_langs     ,	max_position_embeddings=self.max_position_embeddings     ,	initializer_range=self.initializer_range     ,	summary_type=self.summary_type     ,	use_proj=self.use_proj     ,	bos_token_id=self.bos_token_id     ,	)

              return (
                  config,
                  input_ids,
                  token_type_ids,
                  input_lengths,
                  sequence_labels,
                  token_labels,
                  is_impossible_labels,
                  choice_labels,
                  input_mask,
              )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  TFXLMModel(config=a__  )
              _UpperCAmelCase			     =  {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
              _UpperCAmelCase			     =  model(a__  )

              _UpperCAmelCase			     =  [input_ids, input_mask]
              _UpperCAmelCase			     =  model(a__  )
              self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, self.seq_length, self.hidden_size)  )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  TFXLMWithLMHeadModel(a__  )

              _UpperCAmelCase			     =  {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
              _UpperCAmelCase			     =  model(a__  )

              _UpperCAmelCase			     =  outputs

              self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.vocab_size)  )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  TFXLMForQuestionAnsweringSimple(a__  )

              _UpperCAmelCase			     =  {'input_ids': input_ids, 'lengths': input_lengths}

              _UpperCAmelCase			     =  model(a__  )

              self.parent.assertEqual(result.start_logits.shape     ,	(self.batch_size, self.seq_length)  )
              self.parent.assertEqual(result.end_logits.shape     ,	(self.batch_size, self.seq_length)  )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  TFXLMForSequenceClassification(a__  )

              _UpperCAmelCase			     =  {'input_ids': input_ids, 'lengths': input_lengths}

              _UpperCAmelCase			     =  model(a__  )

              self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.type_sequence_label_size)  )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  self.num_labels
              _UpperCAmelCase			     =  TFXLMForTokenClassification(config=a__  )
              _UpperCAmelCase			     =  {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
              _UpperCAmelCase			     =  model(a__  )
              self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.num_labels)  )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  self.num_choices
              _UpperCAmelCase			     =  TFXLMForMultipleChoice(config=a__  )
              _UpperCAmelCase			     =  tf.tile(tf.expand_dims(a__     ,	1  )     ,	(1, self.num_choices, 1)  )
              _UpperCAmelCase			     =  tf.tile(tf.expand_dims(a__     ,	1  )     ,	(1, self.num_choices, 1)  )
              _UpperCAmelCase			     =  tf.tile(tf.expand_dims(a__     ,	1  )     ,	(1, self.num_choices, 1)  )
              _UpperCAmelCase			     =  {
                  'input_ids': multiple_choice_inputs_ids,
                  'attention_mask': multiple_choice_input_mask,
                  'token_type_ids': multiple_choice_token_type_ids,
              }
              _UpperCAmelCase			     =  model(a__  )
              self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_choices)  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.prepare_config_and_inputs()
              (
                  (
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				
              )			     =  config_and_inputs
              _UpperCAmelCase			     =  {
                  'input_ids': input_ids,
                  'token_type_ids': token_type_ids,
                  'langs': token_type_ids,
                  'lengths': input_lengths,
              }
              return config, inputs_dict



@require_tf
class 						lowerCAmelCase       (   snake_case      ,						snake_case      ,						unittest.TestCase				):
       lowerCAmelCase__    			=	(
           (
               TFXLMModel,
               TFXLMWithLMHeadModel,
               TFXLMForSequenceClassification,
               TFXLMForQuestionAnsweringSimple,
               TFXLMForTokenClassification,
               TFXLMForMultipleChoice,
           )
           if is_tf_available()
           else ()
       )
       lowerCAmelCase__    			=	(
           (TFXLMWithLMHeadModel,) if is_tf_available() else ()
       )  # TODO (PVP): Check other models whether language generation is also applicable
       lowerCAmelCase__    			=	(
           {
               """feature-extraction""": TFXLMModel,
               """fill-mask""": TFXLMWithLMHeadModel,
               """question-answering""": TFXLMForQuestionAnsweringSimple,
               """text-classification""": TFXLMForSequenceClassification,
               """text-generation""": TFXLMWithLMHeadModel,
               """token-classification""": TFXLMForTokenClassification,
               """zero-shot""": TFXLMForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )
       lowerCAmelCase__    			=	False
       lowerCAmelCase__    			=	False






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__  ):
              if (
                  pipeline_test_casse_name == "QAPipelineTests"
                  and tokenizer_name is not None
                  and not tokenizer_name.endswith('Fast'  )
              ):
                     # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
                     # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
                     # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
                     return True

              return False






       def 				__A			( self  ):
              _UpperCAmelCase			     =  TFXLMModelTester(self  )
              _UpperCAmelCase			     =  ConfigTester(self     ,	config_class=a__     ,	emb_dim=37  )






       def 				__A			( self  ):
              self.config_tester.run_common_tests()






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_model(*a__  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_lm_head(*a__  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_qa(*a__  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_sequence_classif(*a__  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_for_token_classification(*a__  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_for_multiple_choice(*a__  )






       @slow
       def 				__A			( self  ):
              for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                     _UpperCAmelCase			     =  TFXLMModel.from_pretrained(a__  )
                     self.assertIsNotNone(a__  )



@require_tf
class 						lowerCAmelCase       (   unittest.TestCase				):






       @slow
       def 				__A			( self  ):
              _UpperCAmelCase			     =  TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'  )
              _UpperCAmelCase			     =  tf.convert_to_tensor([[14, 4_47]]     ,	dtype=tf.intaa  )  # the president
              _UpperCAmelCase			     =  [
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
              ]  # the president the president the president the president the president the president the president the president the president the president
              # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
              _UpperCAmelCase			     =  model.generate(a__     ,	do_sample=a__  )
              self.assertListEqual(output_ids[0].numpy().tolist()     ,	a__  )






